diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee1aaf463800824b181c45f9f4cbac5ae05d1eef
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1295929648118789, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.017717451610532654}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.057688485079104755, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002659417350231105}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.18771361409144396, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.004194847117781067}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07130151635044231, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017145172534413066}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.022591789424650413, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012409453957858067}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.08711168383171403, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0026849545852757613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.031734228703201264, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010224014054478407}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05685991375829496, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0026473311931234454}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.18500805358362152, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.004147431246244616}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.0701415239947909, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016880620041233863}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05595104569982353, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002636691350690872}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1806575902205997, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003940723272696618}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.068682573498121, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001637051783542573}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ab5173b327a81d4ee90c70e0028fd5c52826580b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.14987013983247244, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01941974811684623}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05507727706640666, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019564677497519853}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.18027308616485552, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0038002016718867975}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07380028517409874, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017644455999507468}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.020536550124082592, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009198527068768572}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07773785714938754, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0024576072009639223}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.029924776302610284, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010166921536067964}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05266527434294661, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0018768852497859027}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.1755608787259102, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0037271693076169517}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07089555573728011, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016677661366070384}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05264662810449602, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001884149706620751}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.17473281354464437, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036722904268852133}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07071172213779651, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001659838827248763}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..501cf0f257ee63db73675c5d886a9b9d288ef298
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.12302264952010052, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.008473608033646678}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.055679194052293414, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015790115772431718}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.1808141666654767, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003768554209591496}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.0768642756778079, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001811200981366701}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02070146702208313, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007492369519392376}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07660233137719231, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00242307758789654}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.030198905160389044, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000982928967923372}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.052039959674758364, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001392609794321448}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17496136681829913, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003685600912041933}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07286820570794587, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016717956031784844}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.052645110361589434, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001455115958473737}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.17488088925366627, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003655243712914895}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07325502354058079, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001694053285569078}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d92f067228ca82b9c67a7fe4715c6b88f99c9864
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1494775278828044, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.022242668837387642}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05819590767349479, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00187266941961563}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.1837211769068863, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037631739340242853}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07759644830725707, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017814677119907972}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.020971065102809404, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000895456781665476}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.078084178799313, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0024146049927506823}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.0299181905955771, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009706547160371815}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05435903919389146, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001720611632601612}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17679767348251002, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036809106852510385}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07316137688753947, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016471496270948857}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05492316378292939, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001744093415859728}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.17690363183304764, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003638138098577823}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07366095409257907, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016586575850230274}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..44de2ce57643f01cea0425cb6186c791761a8cf8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.19164606319365804, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.024842965498983755}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05993715403649896, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016748088373488277}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.18999825922564922, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037155688461071694}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08218073938962703, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018499322659086194}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.021877668010459907, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007551954975307327}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07987598814280476, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023972171750080043}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.031639571635299846, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009862002820496016}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05569962429765539, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014542809726584018}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.1829897393194941, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003626975042403296}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07749057699272385, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016921799932615024}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.056374413163303896, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015045460779383418}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1831731672325477, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003597143627516616}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07799095859947036, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017129521717618506}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3add21234d214016f5dd09b559382588a71663eb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.17574933687846614, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.025384181870486397}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05972888483104343, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018329849074038712}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.18292682744906683, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003663997330795685}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07946226594400384, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018412019107524114}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.021074770698868366, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007683455677750384}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07513861469355282, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002333692457604091}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.030022553056029706, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009742418086937908}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05516127047546632, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015392382782714019}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17614313937858433, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0035646877950734107}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07483720907439476, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001672817823692251}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.055368021748712734, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001561582516111846}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1756797982007548, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003531535198778665}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07489823136944627, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016865681248032596}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f4fa5f27998357ea48bba7af7c60bf26e942e687
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.02716794991360227, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0008204423792394508}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.1822527382056869, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002246189358522882}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.04370443194320921, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008270949225797383}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.001333594787639512, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00011638013641618982}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.009285831703201557, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005095411845886788}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.0021022925642461623, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00011341026500721504}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.026796470589552018, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0007542828958134997}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.1812832654319008, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002242575337885422}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.043375077921510685, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008125955785869596}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.019859301830203156, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.000705291344078074}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.13504875184894552, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0017157651520892864}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.031579687187122944, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000632968954798297}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.007162912027319248, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 8.799732134630633e-05}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4357219369d4b58758b869dfe58f20835271239e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.16883673090259363, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002502030010371229}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.15618300732830487, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002359067976848311}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.14164576016651942, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001778754222107073}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.01672633503624124, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011081302796613816}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.01600514703432178, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009715621757517813}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.013624757551384124, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007818220844644996}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.14224381757445975, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00213034490021511}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.13192814705914896, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00200227535288082}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.11830599709144449, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014079740228845011}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.15042428668427457, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002260006746662012}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.13825539582157836, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002041174554406684}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.1252632960833956, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015337180547933837}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.6590131747530226, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12977618045440018}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..764fd922f7a64d84548604b64bace6794f173318
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.1705969694142156, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025131229098136585}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.15761321437356865, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002432157802246827}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.14354986777463272, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018339278799392887}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.017122190675118196, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012447818915600876}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.016763144168686534, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001041171249249909}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.014100180203474038, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008276615210277857}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.14270089197334904, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002135234458917749}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.1326975552677369, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020774353171695834}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.11949064473731222, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001470601686701589}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.1515024612128631, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002270510371981427}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.1392146312336204, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002115600656748789}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.12656168844098892, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015798154653780215}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.5492689650034025, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08789814958865551}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa8dc3abfc858a9c7971b2b96ea5d5acfa1661d1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.16990770521290732, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025438057474100073}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.16021997486913941, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024160108734416854}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.1441015790918314, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018602827624463706}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.015214536900019273, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010692205053183784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.015420185258765253, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010273736824902339}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.013249922136095603, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008649946361473154}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.14196899698534282, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021319224801637135}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.13510011259495838, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020902509105043673}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.11986132557556263, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001509826051165344}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.15080440042366344, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022898520672142444}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.14098767187068265, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021167327935460206}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.1268605469148488, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016208332554181627}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.5225086157392205, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08782464964651336}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b5ece9e5b57aa94cc4c2f6b43f0c3fa57017cf37
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.17001348002992608, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002472945726795613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.16641335439404173, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002516334305490709}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.14606482836681436, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018280556446259094}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.01591640786398681, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000961538418662567}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.017920512300572296, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001111692174103912}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.014229702953664455, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008237983700670122}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.14223869415288778, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020766661579404897}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.14080246405083494, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021890583449121997}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.12175024842032746, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014788274431685848}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.1506981914188859, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00224238363239455}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.14600473949937198, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021735849443758988}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.12836641711166125, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015963226153607253}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.5342732499256956, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1027315904453386}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c265ec07e0f5cab3a5f5b588fbc869d0486c641f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_precision": 0.17335244138244874, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002628810045816908}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_recall": 0.16573763442748227, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002438138421190895}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge1_fmeasure": 0.14725933348138234, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018907376621132115}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_precision": 0.018051903585621448, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011163727106672703}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_recall": 0.017828395181452815, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011751907098589418}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rouge2_fmeasure": 0.015163133405164434, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009412775453260759}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_precision": 0.1443998895689471, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021924349422918392}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_recall": 0.13967945448595898, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002066115416965032}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeL_fmeasure": 0.12234405519301784, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00149721265164285}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_precision": 0.15358706687881649, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023486595525211425}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_recall": 0.14542801653317822, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020859852388272707}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "rougeLsum_fmeasure": 0.12944852616669666, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001605797602788086}, {"task_name": "GEM/web_nlg_en", "prompt_name": "explicit-graph-description2", "bleu": 0.5599756287973008, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "afeec167-f75f-4687-a775-1efde7d04780", "prompt_jinja": "{{input | join(\", \")}}. \n\nThe above is a set of subject | predicate | object expressions separated by commas: \nWrite all the information in proper sentences.  {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08624872080359487}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..03ea742e4cfe3d66e4eca2c1a3d3f7731b2a261f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.04787169131286324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01817594262932824}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.05219573618812013, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0036463591354812734}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.2092941055787021, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002738958078725276}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.04686425613621658, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0008219392538504159}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.0023893079304512085, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00017566116680500324}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.020431268269765074, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011345589198703068}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.004097730822753029, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000284378780561254}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.05149194714448206, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0036328394668563605}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.2070204298599423, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026942905482556577}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.04594073467891509, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0007486823064547486}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.044498915692550794, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0036541031800773204}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.15356732693991013, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021296241897594786}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.03389173137151838, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006685380625775191}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb218fa4e5be8b916c7105f59a54f0d3b209ae7d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.6245795597079693, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12364633491228348}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.16788566040200872, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002506066712601704}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.156107822389262, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023580502662707355}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.1408351138429239, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017939649759145585}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.016831623170465607, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001109690767212444}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.016312218393737095, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009932711354024328}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.013747268988959595, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007853529846111664}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.14146179896174868, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00213302407544127}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.1319410650215301, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001993005317327167}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.11764346217525097, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014184921221509462}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.14956397844019953, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022662888334540306}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.13813269733237477, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002042887172507199}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.12449737336986157, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015465935152696479}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..26e200fe36d93844414cd5e4e7a866632e362d22
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.44030050792600856, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08918965531864674}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.165020909705563, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002497799117783653}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.1579071204222324, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024816328723868283}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.14063207331011088, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018732758899568784}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.015020754513726486, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009072131098635255}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.016692167939649796, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010286596188788658}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.013404518942651407, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007787079973307792}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.13749360940884617, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020803253174644953}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.13328479413864802, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002139290960697942}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.11682645738111967, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014963127749698607}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.1464246392027477, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022481093268697607}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.1390216810249052, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021491921792093326}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.1239705878552023, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016213406765116238}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfcbcb7dbdab992ab9048b4ec2af57fa2833f0d7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.4028004322246814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09627992264831524}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.16192215223479117, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025886791288695856}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.1591737175072444, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002676280527776092}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.13807126726930546, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001961867694651278}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.01532497701640921, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00096053634684961}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.01789090329021704, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001138291110706073}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.013694547280363368, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008406341572135305}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.1348196406800519, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021490241206731956}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.13491070126011628, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0023624548127525523}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.11455050831050843, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001572738062460073}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.1434457740106499, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023224644970998463}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.13929199647413618, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00233665463674003}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.12128174476152666, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017094370940622802}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..09698fef8e0c97152a6584e9f9db1d003feab877
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.48038870322086485, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0950744978183185}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.1612689936412253, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002593706132714892}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.16230890102132264, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027329862790525937}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.13898420590680097, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002005636475842531}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.01639943402464418, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010462999278646942}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.019047005353722873, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0012346130129613677}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.01467598496147871, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000943689965606244}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.13528835328741515, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021828816198137323}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.1384797370898447, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024281776883333106}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.11632551379631219, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016469373399055434}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.1433977750531889, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023475946507656165}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.14234860995071957, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023567460311511336}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.12238496569136655, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017501406428895951}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0752134d751044a70d6e0ca25abcc50f1e9c91c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "bleu": 0.45115235727169706, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07183544899400104}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_precision": 0.16524773113217792, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002761577061548867}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_recall": 0.1671900838419029, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028680251377737}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge1_fmeasure": 0.14037246602351963, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002008576466387765}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_precision": 0.01852378604139675, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001108714616606197}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_recall": 0.02208117704754726, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014164098810313566}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rouge2_fmeasure": 0.01573629832661093, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009296010590017547}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_precision": 0.13842446049242282, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0023215511141633505}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_recall": 0.14325804112156512, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002575571482543266}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeL_fmeasure": 0.1174214317896419, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016188198074821145}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_precision": 0.14624932204164548, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0024721564063546987}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_recall": 0.14630209531166793, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002457955701083795}, {"task_name": "GEM/web_nlg_en", "prompt_name": "implicit-graph-description", "rougeLsum_fmeasure": 0.12334767454389015, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "38342608-5cd7-4ce7-b2e1-905ecd7f4c80", "prompt_jinja": "{{input | join(\"; \")}}\nThe above is a collection of relations. Write descriptive English that contains this information.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001735345048925764}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..acb36e8ef137a690b1f4cf38bd90115defb185f1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.024257094385346158, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012542520013724245}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.14438146327799528, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002210767634374098}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.03521658777407273, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001032855438532496}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.0036895014079609267, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006419014802940976}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.01251292918426236, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0012328752315072008}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.004339378512582052, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006544799388018146}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.022267351364883243, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001060310841195819}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.137175853341158, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021144498099671533}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.03281178600545513, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008979493883162579}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.021691614849279027, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011300013638413863}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.1294165016082978, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019946881884391385}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.03131853952441433, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.000904933217930217}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.011683013048448883, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.002299898655396354}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d8786c82f745c4f1988f7d891d0125423565a060
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.166421902281267, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002509536624513244}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.15640224145913584, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023886985315635617}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.14008242251401395, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017910410554211656}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.016574599293167843, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001097074412480475}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.016727604524752332, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010405236023368633}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.013642298532841417, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00077960580664087}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.14022832153745743, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00213486067462865}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.13246983853698344, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002033559600741525}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.11709634674078381, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014146593664078942}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.14818197139020592, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002271024873481679}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.13825385118721117, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002074894761466916}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.12378978270420449, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015474580156464902}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.6162564327220701, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1172533643555817}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f772f48d8e55df835ac0ff567566c168894b2c7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.16319819867642651, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002426464101736119}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.15404042754151479, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002478306564411953}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.13892759709779767, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018265728869274942}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.014912982683458914, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009237093791681929}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.016297943525760927, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010649358289921388}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.013089719060620607, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007726454211608279}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.13741035910079322, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020428354842843533}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.13057150223971264, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021219065821278115}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.11650832190139555, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014759630408007285}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.14553524668113407, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022002606329358026}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.13620255058111988, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021639599169587793}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.1229747664140421, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015963298971135365}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.464666510911833, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08222220573011234}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3803f75a71c28c4a6eb10e3970112ba5b10c0c67
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.16188795072384957, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025606178815772186}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.14779323773320033, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025010089208298046}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.1351641368693967, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019474873765221823}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.015524787502426208, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001042198532546253}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.015185223617908139, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00114853509785443}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.013153025539986579, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008953407702700355}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.13524306101257844, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002145203133780158}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.12387529380771088, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002083956552126564}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.11226025544826589, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015601459007713312}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.14350630370022732, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022925759452389416}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.12941050420184938, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002157229969105682}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.11871298212567302, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016806710065392066}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.5347923980216038, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05654660025438367}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba12698447aa9ad4e5b492410b1ce06308414fd5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.1643685889453247, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024901757152926327}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.15465227513600494, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025856665354129014}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.13890191242592734, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018984947282986685}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.015811707006913117, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008876824189595942}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.017023747384715765, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001064465606217159}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.013872966659414053, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007622040129882122}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.13761313588982987, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002106009553028678}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.1298443924561633, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002193510884352127}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.11546828539245778, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015269267163462042}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.1458255331490558, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022720293179680218}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.13542456041940654, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0022271994295923335}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.12188337669823847, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016518640850592327}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.447830278815385, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06715101669775739}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..64fa99c1f24a60fe43877f6fc49464c17bdc06f8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_precision": 0.1646198127817207, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002646484323779933}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_recall": 0.1490304755831364, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023689773981536632}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge1_fmeasure": 0.13625801392457523, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00187042901923777}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_precision": 0.01713122779441091, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010058504976427649}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_recall": 0.015592738904389145, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009494920127045856}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rouge2_fmeasure": 0.013595010492421218, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007463780707733565}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_precision": 0.13886708223114388, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002259848022664383}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_recall": 0.12602515814645163, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019898391778165677}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeL_fmeasure": 0.11419438860339491, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015164933018915136}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_precision": 0.14703846057442585, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0024248196702567554}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_recall": 0.13191735359035292, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020719729544053833}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "rougeLsum_fmeasure": 0.12064027783181062, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001640982408038248}, {"task_name": "GEM/web_nlg_en", "prompt_name": "non-explicit-description", "bleu": 0.4758464184876175, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "9415bd8a-685f-4fa4-803a-f09bd47d4603", "prompt_jinja": "I am taking this tabular data, where each row is separated by a \"|\" and expresses a relation between an object and a predicate : {{input | join(\", \")}}. \n\nNow, I will produce a description of the tabular data using English sentences. {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10146555413589751}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a462634481241ecb3c3b3f443d1e871380ae4a52
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.08308142158928723, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014822533733938113}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.4813530094293165, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0052207165316912555}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.1338747913042839, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002030240874957092}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.029535536265787973, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008372816447783799}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.17908532550201076, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.003415407692561151}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.04755242841283524, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011842958136442528}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.07204203263806305, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011304033839566587}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.44548440426035085, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00514873304330449}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.1175214238809207, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015866825442232382}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.06951673475733636, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013120942001119277}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.40195435934423007, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.004355224640972606}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.11176670755192196, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001789832056695336}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.4105317095314889, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.043066710267683765}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..84cc6bff4f64e17a7f2f532285e021d022ebe376
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.1662782777493377, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0025085144836969712}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.16758435156210738, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030206942122441363}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.14107370791796536, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017982597280216904}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.017259872587569457, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011062856087889594}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.022404809879098816, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001426195880760084}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.014903773827883306, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008144412261826056}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.1398613211175107, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002132190748276612}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.14190066206999388, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026252171398360932}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.11763972203198278, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014171066452616822}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.14788595860466908, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022677650604797407}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.1476217188383832, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002589681176386657}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.12442129008035598, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015468255706592889}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.6314175464516708, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12498388493367703}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..415ca4ab00e9ce4eeeba39e0c426e0569e1df861
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.1653712229197853, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002440924984163296}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.1653250915896547, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029135773698179563}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.14125838813948505, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018147591314905}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.01650829521165785, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010046369743166356}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.020600281994805424, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013239499542895858}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.014494882572376312, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008275950679432722}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.13917888276805995, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020659023219378315}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.14043061866293993, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002517638410671949}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.1184115219641423, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00146376498152489}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.1471096818527704, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0022017203919913464}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.14601316919655402, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025452049628687883}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.1247729325662712, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001572783517385721}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.5931779549207392, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09339568633223094}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..add0c63caee3c1ab76d9c3640eae6ce4a9c6cc42
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.16582665337512495, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024208349959723994}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.16255706599105907, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002834223972206501}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.13960615389268183, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017523193011045669}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.015177432476958089, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008927376595589448}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.019082870994391096, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001300150390527605}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.013187050625056523, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007211908914592551}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.13831816790209567, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020310042200119573}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.13727289920306487, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025055915595956126}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.11581175106585567, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013986409084470917}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.14670366020679268, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00216023102430518}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.1428602401569226, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002485596883393547}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.12243760940103116, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014992182881352384}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.613415825073411, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.15363249845233518}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..1aa9d2d787cd2ed5ca7004a3279d8b0c822bc0a1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.16440247158022403, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002408134958644391}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.1672878148244216, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028544677726046685}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.1415749443824625, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018144061370138271}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.016582657589509987, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008859080182706377}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.02110856663946593, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001326962436284589}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.014802883130151939, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007502428586822482}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.13771889673883428, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002017980024471294}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.14130600028610976, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024909401470214177}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.11783817259815357, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001441797703688301}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.14564352836992275, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002182392381771102}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.14647550822693875, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002455629059429704}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.12419545819826318, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015690813210837236}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.5374477526028321, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09283692815634105}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ce3e3647375db5e318a57838ee4ef468cbffb39
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_precision": 0.16597024457966064, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0026243195088935236}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_recall": 0.16986441790095472, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030299534520702256}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge1_fmeasure": 0.14047911280614023, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001918909621032034}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_precision": 0.018689456645170934, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001121139158372213}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_recall": 0.023799423953292702, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015503997097516637}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rouge2_fmeasure": 0.01624180543052662, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009504654937149193}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_precision": 0.13981148514085578, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002224344509680242}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_recall": 0.14559045715327285, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026961659834251164}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeL_fmeasure": 0.11793359101554969, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015470077154010478}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_precision": 0.14803607615354483, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023878068281125613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_recall": 0.14963478549748452, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026025633576893647}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "rougeLsum_fmeasure": 0.12410304209348472, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016642202118066153}, {"task_name": "GEM/web_nlg_en", "prompt_name": "very-explicit-description", "bleu": 0.5504965404148479, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "426b682e-e801-4e8d-9ac3-5b676c9d3da2", "prompt_jinja": "A semantic triple is the atomic data entity in the Resource Description Framework (RDF) data model. As its name indicates, a triple is a set of three entities that codifies a statement about semantic data in the form of subject\u2013predicate\u2013object expressions. (e.g., \"Bob | is | 35\", or \"Bob | knows | John\"). \n\nA graph can be formed from a set of these triples. An example is {{input | join(\", \")}}. \n\nWrite grammatical text expressing all the relations succinctly and fluently.\n{% for i in references %}\n  ||| {{ i }} \n{% endfor %}\n\n", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.080427930138377}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ed78eecd2efc51a062cf300d6ffbc9baa485276
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.1890231240274845, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019684320777096703}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.339017718024998, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0028323433133959483}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.22563848251054694, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018751473618148513}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.04380838750833082, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008419429245577659}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.08179849168709151, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017042047474535047}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.052567975188848405, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009556547743829917}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.127676216716478, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012153296324758198}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.23953125612828238, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022402139432793856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.15451828388176664, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001208772883015503}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.17481696246700595, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018160527235596866}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.31443864114060294, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026582156582863913}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.20883538497621398, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017334509196587264}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 2.3244260259071865, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07286281897100536}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..9dc891e519fc0bc06962138dd50f19f615fbe1a7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.16030602957044252, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001967524581211362}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.272400210100343, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003051643168753336}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.18474230303970723, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019632536598955266}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.03166192462922729, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007933357990075433}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.05886551805358644, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015884469531505948}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.03761257717991928, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008913626176034577}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.11291539782776717, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012748009948283148}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.19815104405287912, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002316563028814676}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.13089239358773688, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012729550476269687}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.14915402681793893, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018104204962086678}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.25378286760463403, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028305419282366193}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.1719442252457088, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001807476862548232}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 1.828441133091315, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04797244163227974}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6fc7fac8bdc8cc046d60fe42446b71359b11f2f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.13691057056766903, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001825265602905107}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.22539218166685782, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002822899330851212}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.1553013724184807, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001829739118077522}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.02181756499211645, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007137545292173706}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.039804717049478786, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013560991274283968}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.025570973012891195, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007692752377271742}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.10068399069926871, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012243521909029603}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.17123882520643308, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002168643066220908}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.11505946082280019, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012278934893675208}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.12780542015362023, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016861150014377962}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.21034780895799102, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002605948309335639}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.14484893653987274, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016765021056686803}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 1.319957729947558, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.055949786634926266}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a01a469d9b68401894b5fe268a68a590bfb3bcf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.11331660679440218, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020148980956489313}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.17682573132355542, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029418843134879813}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.12309846747867732, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019272106431180854}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.018033312403096684, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006783067936208244}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.031206199578028934, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001235176208046927}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.02030504236701163, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007026247733576691}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.0867545256195891, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015118580414631715}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.13772453378421665, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002277790929230465}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.09410057985901603, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013756456156665128}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.10596910703751164, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001891907064541939}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.16492851491383875, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027267448285863516}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.1147938850890483, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017791199568731235}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 1.2271649835407419, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.061326948665848736}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b66dedb8836e132770fc316e2e5cc15cc2ea30a6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.038476995951992485, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016453155252656947}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.057681790969237874, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023058359041284843}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.039273927677435934, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015037043115189227}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.006055290873763252, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004556065968758473}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.011582702540716703, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0009372131400483686}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.006731993501111477, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00045966773056801204}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.0299699971593133, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012852804923565745}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.04584513362616462, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018409273324060942}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.030451500169627634, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011231733238070457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.03588852826032929, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015400770868070532}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.05364153027915661, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002137335190836827}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.036456484857113516, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013830180281845924}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 0.21901783251929063, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.023129015503409796}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..59ccb984855bac334b255dd49b5ff86d170aea2b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_precision": 0.00741304726602641, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0009755620107809973}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_recall": 0.009480629169855271, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0010572884760238043}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge1_fmeasure": 0.006323793453713114, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000679766707607541}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_precision": 0.001214496073168751, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00022011821368556766}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_recall": 0.0021696621911090567, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00045950435492140597}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rouge2_fmeasure": 0.0012665122562968891, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00021596794467997575}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_precision": 0.006028681674848774, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0008454472955138656}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_recall": 0.007771898833374424, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0008802017587782391}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeL_fmeasure": 0.004971201029477744, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005078524339863577}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_precision": 0.007036952939226733, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0009413610021170078}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_recall": 0.008965157586393747, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0009950047096009401}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "rougeLsum_fmeasure": 0.005929594045281956, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006291046983741132}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "article_summary_en", "bleu": 5.322880476060286e-08, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "2038df7b-5420-4a33-87ec-09715419deef", "prompt_jinja": "Article in English: {{source}}\n\nSummary in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.5191889091292856e-07}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9716e3a673b5d85fac691ecbaecad40dce3cd8f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.07162315852179309, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014555411662114063}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.1024819369868203, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0019137149698756734}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.07728270900364194, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001418281974629439}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.007210468394688485, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00037775068659526225}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.010657999963685463, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000602849822298443}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.007854289251701565, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00040869938896478014}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.06359760111567767, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012025425436729914}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.09274514173812211, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0016759276752536403}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.06907689906047816, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011853028745420664}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.06737233028287096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013379489246377642}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.09693941896958716, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0017781152777598497}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.07283735339621857, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013034613782005062}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 0.4367609232393782, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.023336487096873623}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..911832b3f15f3f56ccfbea060f523a77e201c651
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.10962094154854289, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001521175658021088}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.11614995439958446, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0016222282716261732}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.09906222042576335, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012277637555703933}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.005375616655817727, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00032065018661247955}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.005997760028548002, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00040011451896884595}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.004986359493861402, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002877838903421649}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.08788884421952135, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011845230529361117}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.0930440125082319, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0012664554630627011}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.07875322436499038, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.000899325603615052}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.10514951471452734, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001446840001157129}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.11139163899995776, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0015440273459916564}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.09495364214558669, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001162653836627394}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 0.33862715230568985, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02103608050658023}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d22cf363a568bbbb0be66ccc30fb1473c9a6f5a7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.11008815005619565, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015367268621741792}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.12364976087148276, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0017477601358293203}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.10124369006753632, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012431775944759696}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.006329778123956924, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003579493269657872}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.007946063514698743, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00048230591130947426}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.00600392083872442, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00031876702476632513}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.08810707825607618, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011970905402063467}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.0995681437810585, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001372354219556691}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.08055079492734035, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009086151054529219}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.10550757799818274, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014666803119605407}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.11816997810521175, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016494951956388857}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.09678091628110796, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001168106008335189}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 0.4064700344068096, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.041071462910743026}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4ea9d99e38d7ae088616ebb8e8e87928bcbd1a5f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.094117032806795, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017305123351559388}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.1061886960885055, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0020438338074463868}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.08498062296250532, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014212404882767398}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.006623081561306862, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003792072547215317}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.00957912154003632, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006732767370116441}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.006494830300354191, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0003529124882286109}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.0760422274749195, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013874920699279642}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.08570210788487924, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0016287020168935214}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.0677917896388117, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001061516966060374}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.08992646673675982, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016454052932894198}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.10125000826231054, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001933208130218866}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.08096977315853292, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013335904643873365}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 0.5087314599386397, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.061212421752417735}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..5b288e21b037fcd320e243c199622c8a82e588b0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.033017953252287624, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001455074891420513}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.035407839416534886, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0015340457724425114}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.027959665353093514, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0011329907447503772}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.0028089869136647076, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00035883040231905175}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.0033990005640009368, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00043264989107393107}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.002417994702779381, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002673599459103152}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.027079006620999064, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012047423398949}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.02913572117563057, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0012482614882508539}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.022633019178426736, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008874765902663727}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.031138458647991887, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013669267038162584}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.03352557536851953, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0014450564773056973}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.02640979987744244, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0010615456146286408}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 0.05513061350682931, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01363489004556789}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..186890df8c00b9940af06316c6c03e1de8a1dd62
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_precision": 0.004907621925869614, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0005763406359517721}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_recall": 0.0053768195418473184, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006637161154816707}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge1_fmeasure": 0.0042516054308208645, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00047637308191102623}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_precision": 0.00047314584408024164, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00011906110853937211}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_recall": 0.0004964345162201845, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00016880696249774884}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rouge2_fmeasure": 0.00036529417385801794, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 9.087182991829831e-05}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_precision": 0.0040961836397586295, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00048820210285777777}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_recall": 0.004329727791926256, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005262040146135757}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeL_fmeasure": 0.00343786505841611, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0003771726212323476}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_precision": 0.004702148837329425, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.000555612812542979}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_recall": 0.005058029292746475, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0006171534300008613}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "rougeLsum_fmeasure": 0.004022188513290089, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00044739410245404963}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "rephrase_en", "bleu": 1.9107654017640184e-11, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "753f0a46-aeff-4cd2-932c-8548897cebe5", "prompt_jinja": "{{source}}\n\nHow would you rephrase that briefly in English? ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.3525115607349213e-09}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..c89edcfefda0cf38b158c5d823faacb2b1c5c255
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.04418667472732478, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012197029509449378}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.07215894137790155, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0017665079751954936}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.050630622435102954, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012740642440480783}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.0033950280387810075, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003033243161776345}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.005381920499729423, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00048448492307215835}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.0038458659144674466, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0003374984459731064}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.03927568004870594, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0010304820118418286}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.0651799283564036, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015341625257454653}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.045212023379810784, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010775987760050663}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.04166366640338202, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001132597373996381}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.06842472788539408, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016398413239512239}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.047792067497451136, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001175048993772459}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 0.2645947287972745, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.021907771840493258}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..131f004c1e062bdab95a51cbbf38686b999e9998
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.11065265004766811, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014995222918311947}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.10967982839934531, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0014540559708165963}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.09711652466721561, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00114856823101296}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.004476358135369021, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00028847561060159696}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.0044039325000597256, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0002972809876903113}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.0038867114952000873, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00023684798185580298}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.08933483124530466, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011884320054885904}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.08853836595721083, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0011479891985354547}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.07784452056813264, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008619620021596947}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.10659695950864206, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014326181370186257}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.10579025460545817, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001398522609494789}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.09355863188288549, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0010951306163491682}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 0.26189100567433143, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04164591416025924}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..0b1c531bf5e07167a95af47d68a17ddceb61d0cf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.11362728657731791, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015714871657739951}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.11607779218894242, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001575602946003843}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.10049145155089874, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0011946171315062168}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.005836029460495955, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00036095981618922905}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.0062244104824258515, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004249628756393917}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.005194947294074261, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00030474043998879574}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.09152414632340847, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012419260737023521}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.09395918617125179, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0012571326025089399}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.08052988976365942, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008954855289212289}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.10951406198494446, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001514165066763457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.11185509288980913, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001509586047459457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.09676724655543643, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011401063414987313}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 0.382732965383335, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.041457097975708616}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0caf2358c3236f3e3210736075d41aa018278fa4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.0955940850522753, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018130428010804185}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.09383021551741794, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0017574878214917625}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.08043785817220254, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013311625960759211}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.005894324763379858, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005451846101275158}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.006111381952443814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004988935962431043}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.00471973086085565, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00031285101854069914}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.07845129854611516, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001504183246143632}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.07671122424431512, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0014337379247573415}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.06517286449766353, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010214075492220697}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.09173213439305535, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017274967484957875}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.08992169460486722, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016733690010191946}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.07705416712954397, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0012575955853530694}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 0.3919914059035021, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06209565085248539}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8e8a1cdc7696145b31bd2cbee41f2628a89391ca
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.031069492435776818, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013777037826483293}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.02923924720251318, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001332321685582368}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.024625870173260112, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0010164821198364775}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.0022741707476197034, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0002784957287667364}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.002541556245852753, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00034331230832384933}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.0018715335775846497, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002104568234283972}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.026118959440308825, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011609305606255281}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.024492403731580736, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0011008059304958085}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.020411284941508113, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0008173009244281204}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.0295178758499726, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013119054474959343}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.027611772675785972, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0012447336136434688}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.02329804778849391, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0009534761646565726}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 0.01607270818304334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0034624809654178012}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2364e0b1ea38813b88c3f730440b91ab63d50e08
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_precision": 0.0046970714312177565, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0005792165335246764}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_recall": 0.004114523129900782, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0005483319592202135}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge1_fmeasure": 0.003521129990818492, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0004036686379505131}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_precision": 0.0002427708680341301, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 7.969113434382603e-05}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_recall": 0.00038997340658384743, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00018348649094998182}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rouge2_fmeasure": 0.00020048952920264672, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 6.865942316355645e-05}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_precision": 0.0041148184629156, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005057838120170794}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_recall": 0.003574300499782594, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00047502389608598274}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeL_fmeasure": 0.00304301527398751, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00034135496557363107}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_precision": 0.004523141975654186, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0005607431680479185}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_recall": 0.00392102854568574, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005176638248032069}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "rougeLsum_fmeasure": 0.0033595582196200113, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0003831160597743695}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "summarize_above_en", "bleu": 4.950641247853647e-16, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "088288f3-7516-4cf7-9406-0e082053bf54", "prompt_jinja": "{{source}}\n\n===\n\nWrite a summary of the text above in English : ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 3.846396789441964e-14}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..443b0b7f73e4d4b5a6e48a57a64e8a65818cb7fe
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.08475422777270436, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017956983368717673}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.136466539852913, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026047032098612134}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09664038969174059, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018523781781760917}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.013532165780802595, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005500871126004836}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.023126949071911686, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.000984728105845212}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.015671748077472996, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006102024889325685}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.07093659031967181, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001414812786895555}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11686852788639902, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0021776574996213755}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08141292643254222, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014662345345116555}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.07902371474904281, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001655136022979042}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.12774835789500472, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024261371686350093}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09028162427912649, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017170249037180556}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.9033237981203045, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06093048704966743}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6b51655f8e545384c10e5fb5ed5898d31d426f1a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.11341569440193765, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015875420885948467}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.11847927588394622, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0017387628767267596}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.10140029000732263, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012698513145956132}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006128471241361133, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003870105678849676}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0076069170914782195, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005734677696249654}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.005755944338368179, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00034584001151602705}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09114212196429179, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012300275412546436}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.09521970565929594, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0013695362241421447}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08090174235356164, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009344918578662129}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.10896112722491796, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015092137229918339}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11363652894302366, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016407183665521966}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09730799225374498, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011957026786911643}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.48568473425226133, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08021116263112237}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..cd34c91085316a7892433a034e9791dc9ad8eaca
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.11925916564183478, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017880379896240603}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13943695708117368, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002116433411707032}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.11151220891599137, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014668379581459778}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.010505863803598392, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005738646736412342}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.014684578264016469, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007964172087895143}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.010504603460980755, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0004959353821993032}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09588263877954578, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013996502096768437}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11375527509576841, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001731363662015009}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08956417926961593, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011103097178602006}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.11339532279740656, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016872131131246602}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1323364741919991, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019843253951356537}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.10583744233386468, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013667666533331855}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.7735409292280904, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04682416335531492}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..68ac37eb03b2c6e2354314556447e015a942bd0a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.10733987691391439, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020889839913980418}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.12476072254733013, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002308247813491843}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09780113881018691, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016273262648512452}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.012191906060064263, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007006501461727509}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.01532295103209109, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008648292695711531}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.01096682747964428, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005407448750885603}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.08876565747817264, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017029372366386352}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.10463111270816423, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019111151657166508}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08074375845019817, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012686172017943162}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.10123055336456097, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019729409528934246}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11748223446367839, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021566918011451134}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09202413237033855, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015127002889302342}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.8642314685339636, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07105912584286404}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9865c4ce98ec657c41eaf04a2396e8a39c454264
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.04130438911538399, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018399622962315205}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.04523262499737334, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018427581724112859}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.03531682035958441, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013697458060726072}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0069981887164715815, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008066183569971981}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.00693668855148366, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006076952741663856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.005041922188233573, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00039035553600462724}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.03459558732097317, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015673695864144438}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.037976749671683144, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015299014996893944}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.029228194708275798, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001099845646230128}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.038549446328862175, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017184540964501843}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.041863381072326546, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016845634789296187}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.032707886197975086, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001248884867759778}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.11591220265958264, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.018810551681001538}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1fe7107bf39a3618844d0d27021dd5520c09745
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.00840179041155278, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0009319251831963879}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.007933117544594833, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0008986804343115746}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.00646829859544361, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006749836942920999}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.001300602211987841, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0002505531879453807}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0015037014195862734, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00033970791079948175}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0010361138600270753, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00019163061331141026}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0069081946455306804, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.000767045855376856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.006517809586183419, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007338789355087525}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0052628072217658275, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005418137104031803}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.007887486215836917, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0008833842095846506}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.007316433171910116, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0008198190943819032}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.006026061380800861, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006299887360163144}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 6.457241109219731e-10, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 4.3951639243907024e-09}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..5127da63b61a7ac3c66615bf8c360646a255dc93
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.09246228962154374, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001943807421175789}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.13805949306072055, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002409786846441639}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.10023372190565902, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017262119489058103}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.012545852280114898, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005482585896654373}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.021241128219617465, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001043761840842228}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.014303497306763413, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006015388106615314}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.07837678044248741, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001609888637751305}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.11893781436000582, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019490781148502943}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.08504732396245059, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001313235354820681}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.08436461190041952, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017072730429511782}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.12779895161379792, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0022186936931219443}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.09216848394827927, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001563789121056033}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 0.745448160166905, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.047547732193019436}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..bda21d53121e4fd004e8c22a88f5218ca328837d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.10990140516594758, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015351949382576508}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.11791561392844088, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0016685835887549996}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.09978220637884393, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012449757420128587}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.005589121653905529, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0003414693806880958}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.006477748246242698, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0004490780641964607}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.0052645090649109895, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00031778114128576976}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.08809126672816353, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011930762685352886}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.0946530891608615, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0013251798935214189}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.07936011949636798, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009144164542617104}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.10565199584207204, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014615877820162168}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.11343373068042586, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0015989418107090036}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.09589920116880182, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011829464434838014}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 0.36679033215823664, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.029277836361420107}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ede2424f7b36b05ae16792b88332b4c33bfb8328
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.11013619429611844, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001546664571183218}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.1232803535134769, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0017517550316112635}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.1016546124922999, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012651922442427001}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.006234351643431517, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00034125509784363245}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.007962049327295925, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005161503037287333}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.006048464517830181, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00032306729820669704}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.0875404512463235, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00118479594104557}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.0982995279262905, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0013504657871243205}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.08023398785251074, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009106005118733982}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.10557342164225539, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014719765283516568}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.1177632536098584, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016421313291817262}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.09722400124042876, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011899218963608768}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 0.3820141559457903, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03590316359310684}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..daa1453d5a3bb0c313859f239881639315f7255c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.08899396863415189, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001777790692308317}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.09692319541629159, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0019946027284699383}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.0785984819107557, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014372932969101489}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.005984354455890461, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00040684067708898005}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.008510439649050199, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006316823704492148}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.0058753493744521336, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00036684689106018846}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.07217146798975321, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014091242615705865}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.07831148959974298, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015814306656436506}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.06287166017244038, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010782265186488856}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.08495497287249752, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016708456442186747}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.09237830202855782, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018778739354038604}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.07485251192796798, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013431259530301976}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 0.5176377216004082, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06866328234036077}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b84edf421d3a990097a78fce543a70e169b3419e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.024653052383932526, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0012369651915557867}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.026978552912746406, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001367418485604499}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.021150352711754538, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0010009476386843301}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.0020847909583644186, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00028781597725583064}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.0025592724240508426, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00032606918523288823}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.0018364476816945278, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00021456601406933008}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.019887895435798185, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0009981749275954795}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.02207509587363469, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0011126244037750546}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.01695587958257084, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0007821077415757568}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.023148124955408137, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001155778717437976}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.025400258275254814, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0012784306268269182}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.019854394029124416, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0009334487521584576}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 0.013454379756596573, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.004377302392144288}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..0da98372b36698d31a0cab65f254a4a73c00b727
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_precision": 0.002594820378060942, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00041260070427087084}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_recall": 0.00315946115980561, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0005463787261087583}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge1_fmeasure": 0.0023404225914027293, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00035897358936479027}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_precision": 0.00020295754731072598, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 6.30448872924687e-05}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_recall": 0.0004422474374560793, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00017450355164885773}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rouge2_fmeasure": 0.0002340501971901335, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 7.27640076429073e-05}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_precision": 0.002059754228694586, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0003240033862130502}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_recall": 0.002522670751072346, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00043340282734999423}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeL_fmeasure": 0.0018292868173016377, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0002697379131437824}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_precision": 0.002476388701828487, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0003949847874533559}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_recall": 0.0029610416476689777, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005018145536321888}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "rougeLsum_fmeasure": 0.0022043625397931894, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00033340306926608464}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "write_abstract_en", "bleu": 7.304724850661707e-22, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "dff7b314-7385-4855-bb90-253073a34fde", "prompt_jinja": "First, read the English article below.\n\n{{source}} \n\nNow, please write a short abstract for it in English. ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 4.651649551405445e-19}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9416d8f59d48c1b2f7b4acf41ad0f42aa6237f1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.323, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014794927843348633}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014910846164229857}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f91a5e8db2b895c592f3bac754807ecc680aaa1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.33, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014876872027456732}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.326, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014830507204541037}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1954f331eb4b54a40383e6d032d686052a06e96c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.354, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015129868238451772}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.35, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015090650341444236}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..8000d33ff88b88877c71189b4652994d11273c01
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.347, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015060472031706618}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.346, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015050266127564436}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..285b073fc0d77ad85143d728fffea84ce9e2b3b5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.337, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014955087918653593}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.345, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015039986742055235}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc59c5f0b4f9dabb0e450d29a8a8180c30d8e43b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc": 0.344, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015029633724408948}, {"task_name": "anli_r1", "prompt_name": "GPT-3 style", "acc_norm": 0.333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01491084616422987}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e72282b333e8b7dc5ca6e476342c854f3c3baa91
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.321, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014770821817934654}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.334, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014922019523732967}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e0da6d7628e73a3cd0c42b19c8859b52bf3c686
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.333, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014910846164229863}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.333, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014910846164229863}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7a6ddd7b231835d138865424f66d0da13e70d6f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.338, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014965960710224485}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.336, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014944140233795021}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ada1e7e6285941814b904685b5d2babcc77a7b12
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.373, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015300493622922814}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.361, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015195720118175124}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e671120137826c1327b94398e0c32276a2825eb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.346, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015050266127564446}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.342, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015008706182121728}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..8be664a424bd4dc3b65c979e425e7948b4120dd2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc": 0.338, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014965960710224485}, {"task_name": "anli_r1", "prompt_name": "MNLI crowdsource", "acc_norm": 0.329, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01486539538592836}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..3946790f739b8acb69d0a7a1fab22af00fd2736b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.321, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01477082181793465}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.337, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014955087918653603}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8da65c9bcc816f82a5b7af4e8368354789acaa12
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.33, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014876872027456732}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014910846164229863}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5bf6b64e2e20d5ca8828e3b2d2471b573d4fc32a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.342, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01500870618212173}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.343, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015019206922356953}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..1ddb06595d9b61fcfe4409aada901f8659e9da24
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.349, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0150806639915631}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.36, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015186527932040115}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..3390aea0d3fa2f7b671e6a23eadc31764864e3f5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.341, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014998131348402709}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.337, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014955087918653602}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5131f2cd81c2080b7aa65711914ce7313a996a9d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_can-we-infer_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "can we infer", "acc": 0.332, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01489959724281149}, {"task_name": "anli_r1", "prompt_name": "can we infer", "acc_norm": 0.33, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014876872027456736}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e656ce44191c0b08cdee1cb086b161e8027d94a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.318, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014734079309311901}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.333, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014910846164229857}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8bec3719e6ad66cd5bf87fb1e02cfb02acc9733b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.333, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014910846164229863}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.333, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014910846164229863}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ab120faceae0e29833b86f98be32c1abad884c2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.336, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014944140233795021}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.337, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014955087918653603}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..583eb0672f188bbab39bd35ffedd5f4d86b65b14
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.339, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014976758771620342}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.338, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01496596071022448}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d4a8a8fcc0e8935c1a5f5062b10906b0637b1fe8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.319, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014746404865473486}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.326, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014830507204541044}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3185e8672326edea48f15b358de47eda550f5ccd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc": 0.324, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01480686473373886}, {"task_name": "anli_r1", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.324, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01480686473373886}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..38aa45b661c42c0d910e4654f3ec680760da52db
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.32, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014758652303574883}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.337, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014955087918653596}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3cfaae0ca2ea113c8c90f4da151d8e50e3330884
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.327, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014842213153411242}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.328, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014853842487270336}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..9e5ee612cf982d2b300bab1cdfe7ad62cf6a1771
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.346, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015050266127564431}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.353, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015120172605483706}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e46f3844a8ce0a63ec273542dd9bf4c3249b3d03
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.351, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015100563798316403}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.35, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015090650341444236}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..45cf5204d2083a968b7e239f994b808ac272da31
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.342, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015008706182121726}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.347, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015060472031706618}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..398df0cd00446fabc08eb974840249248978bd76
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r1", "prompt_name": "justified in saying", "acc": 0.339, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014976758771620337}, {"task_name": "anli_r1", "prompt_name": "justified in saying", "acc_norm": 0.321, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 1, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014770821817934652}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..af04a346a9a1645c86cff3abaff3b4a319dbf282
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.32, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014758652303574888}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01491084616422987}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e929ccc7f8dc383fe1ec6f9ba1f927ef2ee4c50
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.318, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014734079309311901}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.315, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014696631960792508}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..865b2b7d8c5ce55cb82494c755a6fd5b783e6231
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.329, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014865395385928373}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.324, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014806864733738864}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..aed62b897f3a7baa0e9235522f8fb4d4384b51a9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.325, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014818724459095524}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.317, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014721675438880217}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e20dab8b2cd714bb36cc7bc73349590f04a0042
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.336, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014944140233795021}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.322, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014782913600996681}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..62767a38380eeb1eba72f02b5faf22363d54ff04
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc": 0.333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014910846164229871}, {"task_name": "anli_r2", "prompt_name": "GPT-3 style", "acc_norm": 0.33, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01487687202745673}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..dbb4ff68bbb568b8c900ae1f57e24d90acbedf82
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.336, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01494414023379502}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.344, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015029633724408941}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d885ddcf51645bc671ebef1ef4cd7fb7b76f1acf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.315, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014696631960792506}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.315, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014696631960792506}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f00f6c0741c9c225fe713c061972510303675b1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.321, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014770821817934647}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.326, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014830507204541033}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..008cd5fecc8ed73d03a81717c9224b38f726d027
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.325, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014818724459095524}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.329, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014865395385928362}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ce21a2f48a589ffe7b5447aac4f2bcebf2b61f4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.325, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014818724459095524}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.328, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014853842487270334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac901907c463fb91515bb5912aa7caaed39d9b1c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc": 0.303, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01453968371053525}, {"task_name": "anli_r2", "prompt_name": "MNLI crowdsource", "acc_norm": 0.317, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014721675438880226}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..47acb5a190208e48daa68949178006e0cf608f42
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.336, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01494414023379502}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.332, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014899597242811488}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8aa809c45216260e34fc2a6b4ff998a909a98f21
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.318, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014734079309311901}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.318, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014734079309311901}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..17ab4b1d934c1d108025977e53d70b92c7ef2d83
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.324, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01480686473373886}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.321, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014770821817934645}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..54b203f8103b18a9cfd57eca7925bc759114250d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.337, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014955087918653596}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.341, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01499813134840271}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..be137b959b8ab8653e9a10599d5a101dc1e7b72d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.328, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01485384248727033}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.336, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01494414023379502}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6e266036ec82f3534872583a78dbc1f2c20bb90
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_can-we-infer_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "can we infer", "acc": 0.336, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014944140233795021}, {"task_name": "anli_r2", "prompt_name": "can we infer", "acc_norm": 0.325, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014818724459095524}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa5267a8edbadabbac1d76c58590cf349b9378a4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.348, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015070604603768408}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.329, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014865395385928352}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f60ebd903b82baf5b7fe9a987a1b3cd5cdbe173e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.315, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014696631960792506}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.315, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014696631960792506}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a52a0583fbdb63464537fa4fe0d8a8c18aa24f0b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.32, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01475865230357489}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.317, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014721675438880213}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca99d73810748dbedc9a45d63aaa70fc99e3bc00
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.326, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014830507204541038}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.328, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014853842487270334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..367b67f02667b711221ea88f71273214a95b617f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.325, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014818724459095524}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.32, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014758652303574883}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..13868f6ff730bbaa59b1d0bdd6d7086ce572a7d5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc": 0.317, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014721675438880224}, {"task_name": "anli_r2", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.314, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014683991951087964}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..89a04d085d185c1a10b7979ccb30492dfc370493
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.311, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014645596385722694}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.329, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014865395385928367}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4434d6561a0f273fb2ff20e56fb18a5428ce5e8b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.32, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014758652303574886}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.319, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014746404865473479}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..292d99f74a14a3b966c5b269048c465b8c9fd01b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.311, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014645596385722692}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.322, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014782913600996685}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..de9683ed795b1a449198def1e52a602932379d29
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014910846164229868}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.341, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01499813134840271}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..4c83a5e9428693600ea1ba4b469ef7210099d727
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.329, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01486539538592837}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.323, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014794927843348633}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..650f5be2bf84850463224cefee1372896fa43bc4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r2", "prompt_name": "justified in saying", "acc": 0.328, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014853842487270336}, {"task_name": "anli_r2", "prompt_name": "justified in saying", "acc_norm": 0.337, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 2, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014955087918653596}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..c81c31a1df5e30c148c61268165ff78eee21700c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.30583333333333335, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013306526255831147}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.335, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01363087184382147}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f359ceef557b872f39e34ca93adf179509895f40
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.3408333333333333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013688600793296932}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.3333333333333333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013613950010225608}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b519754bfb0912e831a9795ab72e855d03bdb324
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.3416666666666667, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013696658778002519}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.33416666666666667, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013622434813136767}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f17d770067e8819a7f968fd119f773b9a2282510
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.35333333333333333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01380457216231493}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.3575, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013840921245257787}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf443b0bde21f2a5cca5f9cc8e4ce1e2a2b206b3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.3416666666666667, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013696658778002519}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.3308333333333333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013588208070708995}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..4cf8716bb390ecd5c5393c6bec7f06f255897862
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc": 0.3233333333333333, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013508372867300231}, {"task_name": "anli_r3", "prompt_name": "GPT-3 style", "acc_norm": 0.315, "fixed_answer_choice_list": ["True", "Neither", "False"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "620aa3fc-d5eb-46f5-a1ee-4c754527aa97", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013415009084004859}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2da56ce5a17317a523f11071a2d2f20743e4eef3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.335, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013630871843821477}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.34, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013680495725767787}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1255e74f2a44ad2ff7a8ddd0725d66b47b83c99e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.33666666666666667, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013647602942406393}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.33666666666666667, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013647602942406393}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..edea79296192e2685a4e1d739674e48427e01efd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.3433333333333333, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01371263383046586}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.3283333333333333, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013562032919529015}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f5d798aeff43974313edec2820203f62207c1dc
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.325, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013526454480351023}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.32166666666666666, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013490095282989521}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8fe5737973b21f3265fb73237ee884f8030ca946
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.31416666666666665, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013405399314984101}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.30833333333333335, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013336721143136467}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..6db00599f06088e7fa050a18fbbe6fb82b8e8d88
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc": 0.30916666666666665, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013346684134591945}, {"task_name": "anli_r3", "prompt_name": "MNLI crowdsource", "acc_norm": 0.31166666666666665, "fixed_answer_choice_list": ["Correct", "Inconclusive", "Incorrect"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "0cc3ae39-3997-4686-8c93-5d51457efa1f", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013376268790982112}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..abc9849cfeb8cc8632d51f2a670afe7e283e4538
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.3433333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01371263383046586}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.3441666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013720551062295756}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..994b92c53145d73087b4d9a0b9b19f3ca2b161aa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.3441666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013720551062295756}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.3425, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013704669762934725}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..53943ef133fa4e681f46aed21a5b470c35243dae
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.335, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01363087184382147}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.3283333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013562032919529019}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..72d2ff48e4133b4de8f238c3c399932d6e0e8a87
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.3375, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013655897185463667}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.3425, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013704669762934734}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..63e903ae953e8f3cae623535fdac4ea0cd019cc7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.3358333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01363926119093287}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.32666666666666666, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013544340907003665}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..112e814fd171a7a1c0a34f4b4e886010954cfb04
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_can-we-infer_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "can we infer", "acc": 0.3258333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013535422043417471}, {"task_name": "anli_r3", "prompt_name": "can we infer", "acc_norm": 0.33166666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "c4ed37ae-d7d7-4197-a725-ef2152fa3b1f", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013596836729485163}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa60f4c827dc534dd1c04f6d06dd84161eb51100
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.32166666666666666, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013490095282989521}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.32916666666666666, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01357080625843362}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..09970ca9cd798e46cfa1e08852e809155798ad76
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.33666666666666667, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013647602942406393}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.33666666666666667, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013647602942406393}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..cfd4f18d5b897cc89c022d2ec225275def8de032
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.30916666666666665, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013346684134591941}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.30666666666666664, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013316642319070695}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..63c15a1e24cf188877a11608068c6a773f1f775c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.3125, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013386029277441229}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.3125, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013386029277441229}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b00f483a9987b2cc41f72094cbd833197898b668
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.31166666666666665, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013376268790982112}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.30583333333333335, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013306526255831164}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..10ca491e2424d5fd5a80c7ec9c7501ea153df8ca
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc": 0.32416666666666666, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013517438120881629}, {"task_name": "anli_r3", "prompt_name": "guaranteed/possible/impossible", "acc_norm": 0.32083333333333336, "fixed_answer_choice_list": ["Guaranteed", "Possible", "Impossible"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "ca24b93a-6265-462f-b140-e329c03d94fa", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {{ answer_choices[label] }}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013480882752851555}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..8acd3a919da66ddd31655526c4ad5e6367aab4eb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.35, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013774667009018552}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.3308333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013588208070709002}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e315791feef4308d17cff291efaf9da741213dd6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.3466666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013744022550571949}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.33916666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013672343491681819}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..48f829feb2cf6e4094348380732d31e57f5964ae
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.3275, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013553211167251953}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.33166666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01359683672948516}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..bfce0d4237d62cde8d120bbf794da630fbbc4763
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.3458333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013736245342311012}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.3425, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013704669762934734}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..796aecd51ec11e86de0669f98bf4e2bdd3442705
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.33416666666666667, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.013622434813136764}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.3308333333333333, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013588208070708986}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..76bdb1645e796bee93807d4d172ea1ca56f8352a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "anli_r3", "prompt_name": "justified in saying", "acc": 0.3275, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01355321116725195}, {"task_name": "anli_r3", "prompt_name": "justified in saying", "acc_norm": 0.3275, "fixed_answer_choice_list": ["Yes", "Maybe", "No"], "dataset_path": "anli", "dataset_name": null, "subset": 3, "prompt_id": "a850110d-f1a3-49b4-949a-d3bfe9f81344", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {{ answer_choices[label] }} ", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01355321116725195}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9bb54f0d2146a29edcdc0ca858b35c412f1ec52
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.23293515358361774, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012352507042617396}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.23293515358361774, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012352507042617396}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..972cf312994292b41bdb8bd59f9dcf425a47f258
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.24061433447098976, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012491468532390573}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.24061433447098976, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012491468532390573}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb90e4c8bd6a65f25ed17205c3b69e6c451d8074
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.2354948805460751, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012399451855004755}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.2354948805460751, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012399451855004755}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..823d403f1ad7d5e28d945f108ac467eb934fedfa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.23890784982935154, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012461071376316616}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.23890784982935154, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012461071376316616}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca5fb71a7885747488eb0a745945569596b5238e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.2363481228668942, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012414960524301842}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.2363481228668942, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012414960524301842}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..b70d67a2781a8d767bfa579e86ac5f054eb11658
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.23378839590443687, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012368225378507161}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.23378839590443687, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "540ebc31-2ea6-4feb-a6fd-67b6e71cf20a", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012368225378507161}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ade36921b0e1ca08015ef77c360011ec227e4c76
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.20563139931740615, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011810745260742578}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.25426621160409557, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012724999945157741}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f83334f0b8acf708b239957313b6cd93bc01a58c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.19795221843003413, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011643990971573401}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.26023890784982934, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01282193022511256}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..695880268921c50ca8ff2ae69f16251e6b00f514
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.20477815699658702, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011792544338513402}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.2593856655290102, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012808273573927094}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..cfcaa795ed1da3d853009d77058092441a6003e7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.20648464163822525, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011828865619002316}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.25597269624573377, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01275301324124452}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8d64e15937ece31003f347bd136aec436dafe2e2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.2030716723549488, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011755899303705583}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.2568259385665529, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.0127669237941168}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..51a929dd9d24c33441ee26b29d551f23bc2e322b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.19965870307167236, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011681625756888692}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.26023890784982934, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ff84886-9d5f-40d1-80d7-2a39b7c16ec6", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012821930225112556}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0ccf176b32208005ac237ac957a99cd694cf3457
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2235494880546075, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01217489663120261}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.28071672354948807, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013131238126975586}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee2c70c58d0d423a0f9a0b30f18761cca81eed12
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2175767918088737, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012057262020972497}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2858361774744027, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.013203196088537369}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a88f866ed521d6522a23c8849e229fbc64241ca2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2158703071672355, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01202297536003068}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2687713310580205, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012955065963710696}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e92af1f85fbc5c7eab0a9e32e7720acf004aff80
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.21928327645051193, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012091245787615725}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.25597269624573377, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012753013241244532}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ce66c10162f7e61abaae6aae30bf89314fc10094
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2150170648464164, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012005717634133604}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2568259385665529, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012766923794116801}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..9140059227b9e2a8a074d9c35f475d1c6a9c086a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.23122866894197952, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01232085883477228}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.26109215017064846, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "ced2b33b-b590-4522-b041-51d7dd669561", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012835523909473845}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..1f147b45da46eb25cf5222b2269a671e897e2509
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.23464163822525597, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012383873560768675}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.23464163822525597, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012383873560768675}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..2faab998d5200cc2bb573ebc0710b4cb1d903ed0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.24061433447098976, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012491468532390571}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.24061433447098976, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012491468532390571}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6449a80558f8227b908ad57a7b93aa1987ce25cb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.2363481228668942, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012414960524301839}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.2363481228668942, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012414960524301839}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b38e36f1d0dbc20b6231e113a607f9d7284221a8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.23293515358361774, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012352507042617393}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.23293515358361774, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012352507042617393}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a7928488af6975e5d202b7fbe3f8abc2020a4ec
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.23464163822525597, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012383873560768675}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.23464163822525597, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012383873560768675}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..0904e05414c128d62ccaa475c851ebab5feca287
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.22866894197952217, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.012272853582540813}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.22866894197952217, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "e371fc1a-8edb-477b-b345-9d73e97ffade", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012272853582540813}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..51ceb2fe46ac9a688e5b271ee62ce04ab8d5fda1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.2030716723549488, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011755899303705582}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2568259385665529, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012766923794116801}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c44b0c96bf609a3a2750a773320dc425c6cb32e8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.20051194539249148, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011700318050499375}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2627986348122867, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012862523175351333}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6a80389d37818680be3c4f559e3747ef282008c4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.2022184300341297, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011737454431872104}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.26023890784982934, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01282193022511255}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f640cf2eb8c50d57d07592a2ce3c1d05509cea83
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.19795221843003413, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011643990971573398}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2593856655290102, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012808273573927092}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff4b1a4cf726037135c4e9f369611773057b994e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.20648464163822525, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011828865619002316}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2568259385665529, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012766923794116798}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..88b8387ba567fde9806541b4591cb52885dc8686
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_challenge_qa_options_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.2098976109215017, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011900548748047454}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.26023890784982934, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Challenge", "subset": null, "prompt_id": "5ec2b8ca-e4c0-444e-b097-89ccce811550", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.012821930225112552}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..83ce428dbf4019a50708f6ab757ba48a1c778ab1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.24957912457912457, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008880241465504347}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.24957912457912457, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008880241465504347}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..aef095352062ea596f4a8134d33dc418237a3cd0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.2474747474747475, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00885511441483471}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.2474747474747475, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00885511441483471}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..74f291a24effb9260f1ecb5c2a3e47cb4a42d1e2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.2558922558922559, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008953950243013991}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.2558922558922559, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008953950243013991}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..dee54566ef8dd3788c303c5919ff9a9f23040f0f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.255050505050505, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008944265906130709}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.255050505050505, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008944265906130709}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1e5f70eb6dbb0d047a8d4c16f153b2c6071b962
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.24452861952861954, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008819461106822598}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.24452861952861954, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008819461106822598}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..beb2ab7c880f4002a1df7dbf2dce2a886fa66bde
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc": 0.26346801346801346, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009039157374497713}, {"task_name": "arc_easy", "prompt_name": "heres_a_problem", "acc_norm": 0.26346801346801346, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "d90da519-0e2c-4f9b-a546-7cba82824eb2", "prompt_jinja": "Here's a problem to solve: {{question}}\n\nAmong the 4 following options, which is the correct answer?\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n {% endfor %}|||{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009039157374497713}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9e8af29c17740118dbb57db4811173eef24140e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.2895622895622896, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009306838912173909}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.27525252525252525, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009164888895174743}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a271635eaf9a02c5ffdfa60214fb1249153719c8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.29124579124579125, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009322788837938861}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.27735690235690236, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009186490105111902}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..81228e50daf7d2b901c09b8ad7662a32295aab19
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.29797979797979796, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00938504606669487}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.27104377104377103, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.0091209197417606}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..89eb7ffab2badb5f30c831075d378e930bfd28a9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.28703703703703703, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009282621598983068}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.273989898989899, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009151805901544028}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5c80dcab1fa68c7d087578edc22bde98e77a8f4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.29335016835016836, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009342508331708563}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.273989898989899, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009151805901544024}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..96e76cbc4bd6834f124c9af91ad9293e74bb65b5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc": 0.2908249158249158, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009318815921176647}, {"task_name": "arc_easy", "prompt_name": "i_am_hesitating", "acc_norm": 0.2760942760942761, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "4fb13ac1-f770-45ea-b5d5-91ac50b0d609", "prompt_jinja": "I am hesitating between 4 options to answer the following question, which option should I choose?\nQuestion: {{question}}\nPossibilities:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00917355987383526}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0bc357a169e263e9f9978032481fc4d72611090
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2895622895622896, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009306838912173902}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2706228956228956, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009116466166403825}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..add09c1a9dec6c1f29c93133fc6dc002ba6d0812
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.28535353535353536, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009266280584997753}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2735690235690236, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009147424438490741}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..26eb956e79ba50116d18cc37b16d01201deb670d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.28324915824915825, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009245632200075456}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2786195286195286, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009199329195026348}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..352c6b3162b757e85aa2dfc7865542992e263705
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.29292929292929293, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009338583737393607}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2857744107744108, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009270380606981212}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0f1595df2b134657d74c8eff276a1cf3101c5393
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.2946127946127946, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009354224395837087}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.2828282828282828, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00924147277532823}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a31fee70a86b963cbd1c5934dcb70968b7febeb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_multiple_choice_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc": 0.29124579124579125, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009322788837938863}, {"task_name": "arc_easy", "prompt_name": "multiple_choice", "acc_norm": 0.27946127946127947, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "8c689423-880d-402b-8c7d-a1a98c7589e8", "prompt_jinja": "I gave my students this multiple choice question: {{question}}\n\nOnly one answer is correct among these 4 choices:\n- {{answer_choices | join(\"\\n- \")}}\n\nCould you tell me which one is correct?|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00920783814259724}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d502d92b571549e41d454d16bfcf224aff711c05
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.24831649831649832, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008865199020660961}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.24831649831649832, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008865199020660961}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d0d480b8eb6832fa77f8122cb22c186c949af5b7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.24873737373737373, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008870224411653797}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.24873737373737373, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008870224411653797}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..79cd388c573a860397f5bb9bf705718ad1bd9b99
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.2474747474747475, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00885511441483471}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.2474747474747475, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00885511441483471}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7668002432c6e23c63e72fd8e86afc7b8abae58d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.2542087542087542, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008934537681141544}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.2542087542087542, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008934537681141544}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b61e346ed2764d72964911bf563c14b241da923a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.24873737373737373, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008870224411653797}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.24873737373737373, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008870224411653797}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..683ba967960783b65fbf002c62d05e01bcd88e57
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc": 0.2622053872053872, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00902519799172483}, {"task_name": "arc_easy", "prompt_name": "pick_the_most_correct_option", "acc_norm": 0.2622053872053872, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "033498ca-3d9a-47e3-b631-d881ab53b5ad", "prompt_jinja": "Pick the most correct option to answer the following question.\n\n{{question}}\n\nOptions:\n{% for letter, t in zip(answer_choices, choices.text) %}\n- {{letter}}: {{t}}\n{% endfor %} |||\n{{answerKey}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00902519799172483}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..20eafe12633b12d703a84d6fae74a99d570c6aa6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.29208754208754206, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009330705616569072}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2756734006734007, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009169229476542569}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9547072bd178a44d4e31a0caa84b0f471026198
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.2975589225589226, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009381226721815539}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2807239057239057, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009220526174711356}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..713e2d877893b017d22df5772d182686f4ec5ad5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.29545454545454547, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009361987126556458}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2824074074074074, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009237303403479327}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e40f8b9769033dc95f69355337a088c99c2878c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.29124579124579125, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009322788837938852}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2828282828282828, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009241472775328231}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2720afc0476f5e0847e4d5f1c91c4d26d71238c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.3005050505050505, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009407763090599316}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2828282828282828, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009241472775328228}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..bc05a9bc3916cec40a90232f39dfe003bff646f4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_arc_easy_qa_options_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "arc_easy", "prompt_name": "qa_options", "acc": 0.29545454545454547, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00936198712655646}, {"task_name": "arc_easy", "prompt_name": "qa_options", "acc_norm": 0.2824074074074074, "fixed_answer_choice_list": null, "dataset_path": "ai2_arc", "dataset_name": "ARC-Easy", "subset": null, "prompt_id": "252aa566-9482-4e81-aad9-664a9bebd8e8", "prompt_jinja": "{{question}}\n\nOptions:\n- {{answer_choices | join(\"\\n- \")}}|||\n{{answer_choices[choices[\"label\"].index(answerKey)]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009237303403479332}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c6c0b2048bf02bcb49e74355168b5d9cc6cd56c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.422, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009018450207660421}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.6236666666666667, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008846558976258922}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..66d76bf23cae986ff1c42b37922a637b85843d88
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.539, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009102414587191052}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.5456666666666666, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009092070195065414}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1433726e899fd668e04e1005dad6ec40d9e7f881
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.5253333333333333, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00911850443412263}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.578, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009018450207660424}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..6495208b0a8c650df89b24a9a8b5110865d6e958
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.5316666666666666, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009111901539047272}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.58, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009012606487132152}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff39b246027eb8de690c2acabd1aa55475c13180
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.5383333333333333, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009103358843448796}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.5776666666666667, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009019409415904176}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cff94472ed93466d810dd57fbc03f6278db33cb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_GPT-3-Style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc": 0.5336666666666666, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009109510391803648}, {"task_name": "boolq", "prompt_name": "GPT-3 Style", "acc_norm": 0.5676666666666667, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "492f0f88-4370-46cd-839b-1de37a55aeda", "prompt_jinja": "{{ passage }} \nQuestion: {{ question }}\nAnswer: ||| \n{% if label != -1 %}\n{{ answer_choices[label] }}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009046234144187917}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e090f63db89944b5c2f96358b64ea1c498aa589d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.6236666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008846558976258922}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.543, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00909640486825282}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..2f4983e1295d57df713532cac2695875dcad136b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.5426666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009096928229880423}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.5406666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009099982269204863}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c6ee7673df2f827b37e7b5fa3445b910f4baea3f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.5673333333333334, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00904706345689798}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.5543333333333333, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009076164124491365}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ae8a7073189371c911709d44f9ce8f1adca9a3a5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.5646666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009053547904033172}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.5616666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009060524549335622}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..030bbefa67a8395f4c836a0c80fcc267dbf0dfd3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.5673333333333334, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009047063456897982}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.5583333333333333, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009067881941319685}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..9453de5b75f25349df70e219a2f437515d09b4c0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_after_reading_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "after_reading", "acc": 0.5746666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009027853030468729}, {"task_name": "boolq", "prompt_name": "after_reading", "acc_norm": 0.565, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "3e386463-1715-4578-9cba-07d11a0d3b61", "prompt_jinja": "Passage: {{passage}}\n\nAfter reading this passage, I have a question: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009052751926300883}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a758823cbcaf377955a28d4bd51763c75da3d4cd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.6236666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008846558976258922}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.3983333333333333, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008939496695192017}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..126d7309c77f854a38968ac8765fd6fa53f28d71
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.546, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009091509877386517}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.5436666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009095345834327865}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..7ea3ecd9bba1cbe1d527d3b904fdf57d232a4deb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.5633333333333334, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009056690207178121}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.5506666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009083233528874796}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..58758150e78505227a3a05cc73dfb7469b380d19
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.5626666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009058236409215862}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.5496666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009085074954912701}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d62767b248707d268655af69490c7010bd625446
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.5706666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009038582451449423}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.5546666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009075496684215473}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..50394fc5e7970e508a638f9676c962f6d5e1dc20
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_exercise_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "exercise", "acc": 0.5703333333333334, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00903944935393088}, {"task_name": "boolq", "prompt_name": "exercise", "acc_norm": 0.561, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "9f4c6b0a-437b-40c0-b467-db4b7218d38d", "prompt_jinja": "Exercise: read the text and answer the question by True or False.\n\nText: {{passage}}\nQuestion: {{question}}? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009062029213030573}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5e6552f02b345e4bb953a0fe888712127eca3bf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.6226666666666667, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008851200156534391}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.441, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009066443632063164}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..820234f30a60f0448b2289e202362829797db25a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.5406666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009099982269204863}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.541, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009099483512819305}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..4b732bba70cd1128173c4a7779c1ab31cbb1e4b3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.545, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009093178503605508}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.526, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009117878695396636}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f9346a4b0d3a3b0273cb6e596a6f2b709cb60793
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.5583333333333333, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009067881941319678}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.546, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009091509877386517}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f009068d8e2331c8de0c53e0ec516b0c3a656670
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.551, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.009082611478924382}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.5406666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00909998226920486}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..24665849686cb5340ab86bb916e9b5ae92b80395
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_valid_binary_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "valid_binary", "acc": 0.5626666666666666, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00905823640921586}, {"task_name": "boolq", "prompt_name": "valid_binary", "acc_norm": 0.5513333333333333, "fixed_answer_choice_list": ["False", "True"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "eb78772c-e81e-4b8a-a77b-b75efd1c212a", "prompt_jinja": "{{passage}}\n\nQ: {{question}}? True or False? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.009081985306932099}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..feb816db5d25054edbfe80e361c20bcafad038a3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.6243333333333333, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008843442555522142}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.6236666666666667, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008846558976258922}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..49b1877af32cb3e3043f5cf41cdd411190a729ea
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.547, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.00908980426158007}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.552, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.00908072059340538}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..74f72287470882c9070f8ac958ceca575178b565
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.591, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008977742846741013}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.5963333333333334, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008959169522662578}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc089db106575a91f49a6d93221485ea3a503128
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.5976666666666667, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008954354670397114}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.608, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008914697075129006}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5ec643e16dbf3099dab18e54aa96dad6bfec8c9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.6023333333333334, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008936959925716909}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.6103333333333333, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008905164372580985}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..143431634f20e821bf4d03ef977a9649bfea5d25
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_boolq_yes_no_question_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "boolq", "prompt_name": "yes_no_question", "acc": 0.6023333333333334, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.008936959925716905}, {"task_name": "boolq", "prompt_name": "yes_no_question", "acc_norm": 0.607, "fixed_answer_choice_list": ["No", "Yes"], "dataset_path": "super_glue", "dataset_name": "boolq", "subset": null, "prompt_id": "7cf7acdf-e3a2-459f-a3e8-2e2d27dd6aa5", "prompt_jinja": "Text: {{passage}}\n\nAnswer the following yes/no question: {{question}}? Yes or no? |||\n{% if label != -1 %}\n{{answer_choices[label]}}\n{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.008918717088507564}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..324317d372133cbf19f5ee11637911fb63d4055f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.19642857142857142, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.05357142857142859}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.14285714285714288, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1db8c30f1a82c7084f827ca219c9a4f4934ba831
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.44642857142857145, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942398}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.3171545968156138, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b34c8ffd8ea0bfe2d8857cf54b1c43810e0a4b26
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.42857142857142855, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06672848092813058}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.271744595274007, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..eab70271ddd18ca2164014ac09b9c146b8f46ae8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0663363415035954}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.24603174603174602, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..4c6eeff902748a8cf977d1ebd03044a0e3ae52f4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.42857142857142855, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06672848092813058}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.261867501304121, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6179e05a0fdbb3cf16105f6108830531c44630b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_GPT-3-style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "GPT-3 style", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "GPT-3 style", "f1": 0.235006435006435, "fixed_answer_choice_list": ["True", "False", "Neither"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "75db2bc2-3caa-4956-9653-13c7dd6255df", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ba0ab9370410b52f3c61f0ceaa88a06d63649c78
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06633634150359538}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.1940928270042194, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5772a917a08628f2a431023fb86f23d20b643e4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.2842025699168556, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..41133fa102d0d8884fba9fe84f9b0e83210429d6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.27010148321623734, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..deb0729d252fe426c5562300292ebf68a0b4141c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.42857142857142855, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06672848092813058}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.3048219682039154, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..176df6dc726dd759e4763ba5c20f6db6ec196a0f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.5, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06741998624632421}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.3325281803542673, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff9e5ab4a3cebb10693938fed300da8401c74fda
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "MNLI crowdsource", "acc": 0.4642857142857143, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0672477765493766}, {"task_name": "cb", "prompt_name": "MNLI crowdsource", "f1": 0.28451178451178455, "fixed_answer_choice_list": ["Correct", "Incorrect", "Inconclusive"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "bee62bfa-5307-4e1c-97b2-2ad2f7bcb179", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, \"{{hypothesis}}\" is definitely correct, incorrect, or inconclusive? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..82477f87d48913a5ac8dfbc72ab8d462f81b4413
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.5535714285714286, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942397}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.3737373737373737, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..5afd75a59d4121f200e7fa16150db322f7c2f0af
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06633634150359541}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.29363017934446506, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..eed19e383f1694092fb36a7da208d2d061322562
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.44642857142857145, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942398}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.2956393200295639, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..184f8827cc79c063f1d65fb72ff6a9fe595a953a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0663363415035954}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.25302445302445303, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6064171339b16c99ad0c8dfb1a256c2456b14ad
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.2593406593406593, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2eeb14d7ed7b4a8c0e20f2f0104975db24bbf541
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_can-we-infer_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "can we infer", "acc": 0.42857142857142855, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06672848092813058}, {"task_name": "cb", "prompt_name": "can we infer", "f1": 0.2854700854700854, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "2e76cd0f-68ca-4f03-83ed-11cf15b25a84", "prompt_jinja": "Suppose {{premise}} Can we infer that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %} ", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..5aad7318be4f42cb951339c8e11b4f04416c3a33
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.23214285714285715, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.056929390240001085}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.18837606837606838, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..17f4feca9ff29b94aa442013b2b3ec2438c15b0b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.2842025699168556, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..50724da7aede3d7b76cda25675544bb9177c0685
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.2807909604519774, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..764860637f2bf620f77a87964410f1f1ad74622c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.25852272727272724, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..21a1863fdb344800c898b75055b8d1d0ac53eee5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.39285714285714285, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0658538889806635}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.2719797596457938, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..884badc26957e8c00fa7981dbd2b5ceb53aebe27
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06633634150359542}, {"task_name": "cb", "prompt_name": "guaranteed/possible/impossible", "f1": 0.26059456040480705, "fixed_answer_choice_list": ["Guaranteed", "Impossible", "Possible"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "90ab1002-093c-4e54-b48f-626655e36b65", "prompt_jinja": "Assume it is true that {{premise}} \n\nTherefore, \"{{hypothesis}}\" is {{\"guaranteed\"}}, {{\"possible\"}}, or {{\"impossible\"}}? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..973a1325b7520e5cde7f0fb6fbab0ccbd8f43ea8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.5357142857142857, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06724777654937658}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.3671394799054373, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..fafc96d53bffe868129d2392ddbf22939d3d38c9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.44642857142857145, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942398}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.3172825681224338, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a05e6f17278a44db6e0b9fd881b8f80ee656e3d0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.44642857142857145, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942398}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.2956393200295639, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..de16deeb16f930f28c6e6d45d45adeaae9a0a5b4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.4107142857142857, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0663363415035954}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.25302445302445303, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6272141b81ba8bfcab4978d53047c2361e67b5e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.42857142857142855, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06672848092813058}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.27129417747266715, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..00edb9cdb35cf9d0452f643326ce02a45a13a6c1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_cb_justified-in-saying_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "cb", "prompt_name": "justified in saying", "acc": 0.44642857142857145, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.06703189227942398}, {"task_name": "cb", "prompt_name": "justified in saying", "f1": 0.28883861236802416, "fixed_answer_choice_list": ["Yes", "No", "Maybe"], "dataset_path": "super_glue", "dataset_name": "cb", "subset": null, "prompt_id": "5c9b1fa9-93f0-4f82-b9e3-e0967e4d7260", "prompt_jinja": "{{premise}} Are we justified in saying that \"{{hypothesis}}\"? Yes, no, or maybe? ||| {% if label !=-1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": ""}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6a55636743f651d344334a16a42a69525757beb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.6, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049236596391733084}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.04988876515698589}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e24d9b1178f8ea3802cfc28ac78359e0d405f1f2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.050251890762960605}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.55, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.049999999999999996}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..be195e2faf5310a310d7e5b85f6a760745dbe073
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.52, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.050211673156867795}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ffe76aced082a88d7caa899060495dd8dc7ca399
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.61, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04902071300001975}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..743fd52b8971bac6ebdb3c50db528dbea52be540
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.58, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049604496374885836}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.55, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..062d7472d95d682b610afcffcb070ab596b57fb0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_best_option_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "best_option", "acc": 0.58, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049604496374885836}, {"task_name": "copa", "prompt_name": "best_option", "acc_norm": 0.53, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "8ce80f8a-239e-4393-892c-f63dbb0d9929", "prompt_jinja": "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050161355804659205}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3e44c5e57403b605b441ba092f7aa488f9b5482
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049888765156985884}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.52, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050211673156867795}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8b4d8731ad082c12f238e7c14687db2b349473f2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.57, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049756985195624284}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.47, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05016135580465919}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..32f0a75b6058875ea17ba994c402a97f17049c83
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04988876515698589}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.51, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b54d79a2a767a837426d8517e0a253cab121bb42
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.59, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04943110704237102}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.51, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd2d52add0611ecf6df4b6669e50e5febeb86ec
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04988876515698589}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.47, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05016135580465919}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f51060e920395e7b1e554e2163e003034488619
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_cause_effect_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "cause_effect", "acc": 0.55, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.05}, {"task_name": "copa", "prompt_name": "cause_effect", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "a61d8c21-da25-47bf-b5fe-14a8edd650af", "prompt_jinja": "{{ premise }}\n\nSelect the most plausible {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..beb4dc1a848f0372cb77898a363ed040c54d74cf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.59, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04943110704237102}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.49, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a105cc69400d5fff67ccf324b1a089978d3a1014
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.57, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04975698519562428}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.49, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956911}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..f9905c51938ee708670e8d04bce31f004f23253c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.58, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049604496374885836}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.49, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956911}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e6c76a2976efad1692cbbef15ae531e9f56ba06f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.61, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04902071300001975}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.54, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05009082659620333}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9250df3a2a94ce7b92be8f567bfdb354f219f26a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.59, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04943110704237102}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.53, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050161355804659205}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d084573ccaa783420e261a108a9a87476256ff3f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_choose_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "choose", "acc": 0.57, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04975698519562427}, {"task_name": "copa", "prompt_name": "choose", "acc_norm": 0.51, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "f32348cd-d3cb-4619-87b9-e24f99c78567", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} because... {% else %} so... {% endif %}\nChoose between:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0037e08b57ac4e7637cc53a3a099250b651e5df2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.59, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04943110704237102}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.53, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050161355804659205}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1932d78cf9ad7553a3d36b8eef734814705d2f3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.55, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049999999999999996}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ba073d3eca0e5dc4499b1c6911e2c56b7e8baf9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04988876515698589}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.52, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050211673156867795}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4bd87c00cad1eddc9b6c6d38c26ebd718b14630d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.57, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04975698519562428}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..c01f5d2b483823198c298b7e8e17bc044e321131
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04988876515698589}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.49, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d5f9bca38c032352120b60790fca1c46814c772
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_i_am_hesitating_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "i_am_hesitating", "acc": 0.57, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04975698519562427}, {"task_name": "copa", "prompt_name": "i_am_hesitating", "acc_norm": 0.46, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "4d879cbe-2fd7-424a-9d78-3f5200313fba", "prompt_jinja": "{{ premise }} \n\nI am hesitating between two options. Help me choose the more likely {% if question == \"cause\" %} cause: {% else %} effect: {% endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05009082659620332}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd89fcf5a50cfd97e4f2d738a55113454155ed22
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.58, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049604496374885836}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.49, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..104b8f2c152a13c9a366479594ff2036f7c7916b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.04988876515698589}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.51, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.05024183937956912}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6645aaa2e84c5fd5f1c3ca21ee44c42361ea5e31
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.58, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049604496374885836}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.48, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050211673156867795}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4aeac69138cf7612e43021306d8d81beb6a614ba
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.6, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049236596391733084}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.5, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050251890762960605}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..02c5a9bdb26959c85ec130d0fdd37768e651cee1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.55, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.05}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.48, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.050211673156867795}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5fa43888ecfbc23baa77d68b9dd6aead6df0002b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_copa_plausible_alternatives_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "copa", "prompt_name": "plausible_alternatives", "acc": 0.56, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.049888765156985884}, {"task_name": "copa", "prompt_name": "plausible_alternatives", "acc_norm": 0.45, "fixed_answer_choice_list": null, "dataset_path": "super_glue", "dataset_name": "copa", "subset": null, "prompt_id": "66ea075e-4d03-4a78-b1fa-9a5228cf0c9d", "prompt_jinja": "{{ premise }} {% if question == \"cause\" %} This happened because... {% else %} As a consequence... {% endif %}\nHelp me pick the more plausible option:\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.04999999999999999}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfb2e9c07dc3e69388b3d8aaa66a9067b3e39a6c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 1.5651118840944627, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.042981320396581084}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.18521451898205324, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002457080525787092}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.27275459055962076, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00326239965656717}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.21383961074239713, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0026815356045279183}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.06742918538311436, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012528926958132866}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.0951561319414526, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018453850908383695}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.07683031179945712, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014361637959827516}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.16070278930701914, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00185948984697632}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.24114365421062722, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026802570784209123}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.18671896323697262, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002076974744205931}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.15871609393029643, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0020022541432733187}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.23173214485891466, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026053790108179174}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.1825401908962132, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021610595182852883}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3eed6ebc15ba4ac4e7bcbd64b8807b6bf7b793fa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 1.990645378245907, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09426201816772427}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.19474596234384664, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0036808504078335246}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.17762533937937974, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033135503178414513}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.17658380722184017, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0031889753436656837}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.05236928138258151, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015077160489151036}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.04723848634979288, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013288114505525092}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.04696802025613863, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012839248316351875}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.1402837628671216, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002748825869859068}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.12751328685413893, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002450859209100979}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.12661225264501008, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0023435534892212776}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.16127567452387717, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003143964896409528}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.14650506662968157, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027996830850509635}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.14576477572483743, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002699050142839919}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5a9e987041dd0f2407282068187205e8ddd7f670
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 3.8652987845777536, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10450428506636379}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.2928603436655027, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0033407712098847088}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.26267840583267554, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002974574226301873}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.2629920038581495, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002789298885039384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.08172049965690244, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001717030331861244}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.07276650009557936, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014807864762006147}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.07274573138257315, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014344798818270634}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.21347346743709447, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0026042923393898563}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.1899975461270108, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022474664670417152}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.19024237456492324, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.002100380195663744}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.24332907877165075, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0029356367554022487}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.21726288370114252, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002572105790167164}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.2176826658743973, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0024213633519383406}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..c9c92e4d2265a2ee336f5ee6faf3b84f91cb46e2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 4.749488853686958, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1439737171039592}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.3295110755139736, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030466187495971652}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.2908170186604444, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002713234606538018}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.29307632394991145, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0024677702395856805}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.09691059758420743, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018267761852325394}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.08477572813016075, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015697017492672485}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.08511598673947676, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014976664233839329}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.24119075971078466, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0024282366700050326}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.21098836672574872, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002067666204626933}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.21289346624936825, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018914326319727048}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.274291969853511, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0027222499154415615}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.24075001746941485, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002369489900790716}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.24293590407668367, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002178523714675789}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..892b1b9125ff15a3ce858ce0c512f279400846fa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 5.176301731528647, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.14682428099423225}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.34370313232143146, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002947894794228507}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.3028195228554386, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002584602527686234}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.3042294531279061, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002297980823161359}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.10396227858967162, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0019360112695293232}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.09038662176400038, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016265630560554562}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.0903654887337426, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015331774140601283}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.2524069371766252, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002404139964914726}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.2208918495013449, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0020294042523474267}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.2217428897230324, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017992319739905335}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.28720158613196195, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002676104542866642}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.2520155455904404, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023002075133208935}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.25331485964949535, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020731222812007586}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3d53df621854e382de8b178c927582eedccfeda
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "bleu": 5.448542486611392, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1576603370541913}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_precision": 0.3534259465453393, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002910909198697379}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_recall": 0.31004812770308093, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025222065017164704}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge1_fmeasure": 0.3118801144518462, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002229281342893514}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_precision": 0.11056115709540705, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020146080400812697}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_recall": 0.09523241025254957, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016332379953558076}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rouge2_fmeasure": 0.09565357595328416, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015669516035843776}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_precision": 0.2614140651257123, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002409323599550188}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_recall": 0.22758795096382134, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00197916645130277}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeL_fmeasure": 0.22884790375880742, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017544092909052675}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_precision": 0.29587798141349847, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026666088968318393}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_recall": 0.2582635886545619, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0022422350235981096}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "coherent_text", "rougeLsum_fmeasure": 0.26001996395899674, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "bdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Please generate a restaurant description from the information given below:\n\n{{meaning_representation}} ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002021020490775709}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ddf9633df45e7f84a084879027466cc38d93e094
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 2.604286337933278, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03591674116891077}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.2680956372841705, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0022592902002292296}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.370826122128093, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035950144885036634}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.30133223661624636, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0024851056944864957}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.11163002440463841, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0011657527940780504}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.15654465364549336, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018850260109515933}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.12592500811764892, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013089159002261316}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.20852815533900124, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016092176061611565}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.2840330760519545, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024031245269283963}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.2329565620227209, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001687724446352596}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.2355692548868321, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00191083615835956}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.32127109053431796, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028423630075748696}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.2633236447094561, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020214976908264665}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb482171ffea478eca60af84c1e6cc8401d788f2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 2.376958494986839, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09315943625573786}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.2142240439442094, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003669585143669259}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.19274042573599687, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0032419711801001993}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.19177598419259367, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0031175904013624814}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.05683805777177476, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015258163112187955}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.05073039697609069, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013354368045998223}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.05060323801799519, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012959436966401268}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.1553382299379965, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002742888570452596}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.13961389757181983, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0024018984589737314}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.1385854882700382, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0022957592520056066}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.17863698031283276, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.003158636767078994}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.15979139572794768, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027474819161167415}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.15921566238118462, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002650725044809855}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..cf9b3742657d4d0bb6988941521df48b28144cb8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 4.029805304404254, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.09060124588998114}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.3146026140303434, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.003123666505855817}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.27489622295236277, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027426307774660475}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.27787798890903986, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0025362776452563375}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.08627383555029068, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017285779836978593}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.07473352232906243, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001444402605968691}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.07551724406985702, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001406834249268174}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.23138204856779457, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0025050345781738757}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.1997296995843297, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002078456692172228}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.20227021958830982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0019281583412694655}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.26264908396487563, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002801395123048888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.22749287503171903, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002372211809509306}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.23048345042133192, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0022174670681384323}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac123b8d283758c7ea2d7f04fa197f010c9fb636
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 4.498804726055727, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10775002870645801}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.3356969876826268, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0028462582702245168}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.2941317708277562, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0025456085630560714}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.2968322137022183, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022769902890191806}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.0935059203110401, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017602062083097658}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.0818949185573383, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014958409085964503}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.08225064700727518, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014348926547678262}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.24614076393148132, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002283618951318419}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.21389679836058903, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019512939427380196}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.21588071480185272, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017373391626468349}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.27936714820247016, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025524294154527605}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.24370834239777953, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002246504322912202}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.24604509499852464, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002021644244552477}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3caad14140c2700fdd3c17234d17530fb1a2fc5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 4.927139553620482, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12665338005793328}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.34744584188483457, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0027671133325864244}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.30598680841317655, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024605634807434787}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.30772601506425196, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021433656889168397}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.10081802718812478, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017973283857860576}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.08800270112485714, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015444421661352863}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.08832498480105934, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014651207993424452}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.2559711012561957, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0022826839200581384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.2237171867711202, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001933235277196951}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.2249364821771828, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016844326994501068}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.2906537194445758, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025152309187473194}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.2548745214297313, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021861203156797256}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.2564962774637448, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019326551590934662}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..96220996de6752c90519603aab730d3a69d9f669
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "bleu": 5.319483620066844, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1937918156309555}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_precision": 0.3560891632971412, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0028052847609035393}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_recall": 0.3118566460746229, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0024563894577858706}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge1_fmeasure": 0.3137620150662928, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021267449025118832}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_precision": 0.10787055205478631, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001961259785243488}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_recall": 0.09277219175388018, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001599521459285887}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rouge2_fmeasure": 0.09313015798234085, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00152028521858163}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_precision": 0.2639125125532517, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0023507470257026764}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_recall": 0.22918005986102416, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001931758070692363}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeL_fmeasure": 0.23055494373648347, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001676008041882916}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_precision": 0.2995336579782984, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002598520129287253}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_recall": 0.2608326939117505, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00218886717995965}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "create_text_for_me", "rougeLsum_fmeasure": 0.2627254077501295, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "711bcf63-be82-4937-bdef-0c379d20bb74", "prompt_jinja": "How would we create an appropriate text out of the following data?\n{{meaning_representation}}\n\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019364234716888252}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..21b031a1902ba71fabb37b1fa0875f73cd256b77
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 0.5554048543804149, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04843400111692929}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.07490676533270745, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002150230516202816}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.06832821832134156, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022347623309728333}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.06971399215838711, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002110647717043094}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.04122768230311284, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012955576500527062}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.03832509183940306, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013668800475287908}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.03866825902601929, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001277939861242296}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.07153482505420374, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019220236379015598}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.06510051970164941, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002019905690872576}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.06647105884856924, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018913981605438715}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.07082417678060954, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002048106631259025}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.06475551139934367, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021371018812695345}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.06598366934280134, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002014393055687619}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..96676b7449bc138c30b0bb9c42cf89ddb6bd1d09
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 3.1303501122849213, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.061431179427438594}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.18795241249491457, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019672359252996805}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.1822785883610696, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0021558750964836095}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.17861417240710098, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001841118798739779}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.09099657414603357, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0012717971723587007}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.08699074354785269, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013450179899541406}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.08617713780376496, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001215937184069791}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.1695073282851938, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014742554265132549}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.16297998873383596, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0016527556204933523}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.16060461647433702, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0013748351598955968}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.17092043784842698, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017210240122471166}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.16483164473288184, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001869983093761878}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.1620576843620918, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016013095501406327}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..686635086075ccc989b2146967f97b24cb674c31
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 3.3001775192249134, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10184239170439335}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.2134701829367398, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0024468758692696006}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.20530621471645766, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002460921154929664}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.20055771804060046, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021701685544756343}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.08715705739965049, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0014291170688220639}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.08282189480619162, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014083407755700517}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.08158846564278203, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0012860200593204027}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.18060729349255075, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017814219442781176}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.17257800926899938, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018037731355666897}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.16927544675510284, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015459458010864233}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.18766246584705398, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00211000963150373}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.1797767907757674, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002115732933379051}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.1759439032072785, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018563724554560355}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..50910076939fa22800c560484168221446ea33ba
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 3.666475893279001, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10987537074629235}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.23836199838819472, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0028469068712546316}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.22275349279619636, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002634091620880004}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.21946650594900366, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0024037209981410657}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.08677164701585942, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001628328073416126}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.08034781449611338, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001485529414823335}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.07939963217690697, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013776332563822653}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.1931177821473266, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020670381912188582}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.18001529185227896, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019058042973943167}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.1775636766730472, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016879677128570562}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.20467508975739554, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0024391669746580877}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.19090226151707998, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002261425815356093}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.1882137601760175, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0020477882273726594}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..202a908cadab78b058a805b8b0179e27fa4a5d3a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 3.8809334302287417, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13581356578783566}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.26018033627096865, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0029849403702905605}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.23824263498663878, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0026488517402878512}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.23609371210606708, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002433429420325999}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.08611475286824664, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001635778374326644}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.07858583110015212, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014801568276794114}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.07785610993313638, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013788846901543758}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.20412964784769164, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021807066507413034}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.18679664087085932, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019178081474592414}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.18500764514280799, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017083269070961805}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.22044880753033722, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002598387740639484}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.20134137612920308, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002286825517593027}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.19959413484150632, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002096279733479644}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d7bf67988e552e5da2adde52e3a2a747bf45761
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "bleu": 4.017338372139203, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11094350038266719}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_precision": 0.2695313480190455, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0030112072928501336}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_recall": 0.24782182532367983, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0027049982514322063}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge1_fmeasure": 0.24439453610800832, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002471192570902368}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_precision": 0.08550020741401276, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0016123330184419132}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_recall": 0.07891710085141755, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015002516695171015}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rouge2_fmeasure": 0.07759858672584617, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014022028282609771}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_precision": 0.20874372759839854, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00219378411400417}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_recall": 0.19212524503414735, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019738605195842055}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeL_fmeasure": 0.1891847721343501, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017507687945178178}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_precision": 0.2269415936324815, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00261469191061651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_recall": 0.20793572411416514, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0023144943247615053}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_gramatically_correct_text", "rougeLsum_fmeasure": 0.20519809770842534, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "0f54b6e2-42c0-45ec-8ea2-2e6204388f76", "prompt_jinja": "Combine all of the following data into a concise and grammatically correct text:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\n||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021184619122727845}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a8c3f86efeaee93b0f4875db60c98bc4839e219b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 0.5379698042344059, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.023682890094100795}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.11184434244818633, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020578473204419783}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.17494853797920373, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003281205101820411}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.13249245505272914, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0023865964860820023}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.018039929895774978, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007516851699163001}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.03139481036037028, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014237434012642758}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.02216822438561205, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009397403776169823}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.07937245443292806, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013055330499913698}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.1269667545228582, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0022135782119190543}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.09481463004436187, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001541902659571312}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.10436515332917147, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018952448699960476}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.16392541157802315, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003068122098698215}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.12378548376730361, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.002209146731463173}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ece9c4cdab21e1a2dc1931f7685ce599c507b1b3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.711647887806667, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12239209464334602}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3347678963130461, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002538034125678728}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.27699070131591086, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022531459521912115}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.28795658431823945, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001986436700440943}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.08380014964523355, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015843319722248113}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.06976453338241907, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013297329518253997}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.07230219657399567, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013203982837554223}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2439230596700563, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020005873112574108}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.20072421941893917, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017248535505826324}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2088938861271432, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001534187541306792}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.27665907100653375, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023050480585854313}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.22794132690389687, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001983838704481844}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.2373352341595687, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017982865567598523}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..933c32f266e16ebfdc9f97b38fb813eb1ce1a8d2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 4.158981847733467, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10805920566927497}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3434326875335696, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0026992899622732525}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.2865272615258301, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002262834986590033}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.296355770492695, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002043539678118385}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.09254926113645746, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017008623486038437}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.0763624864031587, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013832865499576604}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.07905711793730946, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013774541250299033}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.25255081340500835, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002168002738840324}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.20966503552217744, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017709941856742383}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.216815482854756, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016012856900651986}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2842764554431881, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0024381683573090783}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.23649066236480792, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002022905140524907}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.244683933643628, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018556926092760141}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..00836cd83b56f40f4b26f07fc3c18cc1775da0da
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 4.7695767170826615, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16549584151536006}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.35625735674057685, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002753457728280898}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.29265236837044695, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002280330795443891}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.3035075966249268, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002031043294823014}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.10577742526833883, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018673396392927667}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.08549991950256323, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014672056887335024}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.08864387470534706, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001446455854434278}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2671981244809395, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002307047675439304}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.21813281187586306, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018116496367021734}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2260702121301555, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016207565003713668}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.29969790030371246, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002551276392318688}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.24481991830895322, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020451634619149594}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.2541669303215731, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018585514707850925}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d41a6a8177e22fb48baba46fcfaee3b6212381e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 5.0994041530025545, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.15184509857797265}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3625581772528726, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002820926829290532}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.29556687359417316, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023030954598190034}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.3069669703069319, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002055596225803597}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.11265300538187106, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.001984841845142737}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.0904636966530554, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001544958177720571}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.09381098739864982, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015224351850222366}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2731899620101324, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00239084885412292}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.22137993241122347, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018642397142735888}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.22975500104331817, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016770985431782879}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.30531248482102935, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026231141349939897}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.24781445089520615, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002101182002409543}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.25753199178665565, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019112760465508692}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..200123d3741923c272a2d8b4d0b34f06c9f08714
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 5.251573482634308, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1553666643851091}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.36700582996097203, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002866361764312095}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.29908671329451914, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022842305263675784}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.31100455847956715, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002052592278425154}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.11715256391592369, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.002004164016012016}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.09355545687055947, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015451237546724788}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.09730358308616667, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001525935012941263}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2792795885938201, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002414534192644447}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.22654007734086326, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018601465471659855}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2352915422950905, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001672302120038204}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3101815290461886, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026513885447339244}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.25160776782553634, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020719183900535583}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.26183889011003636, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001895806329444741}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..571c5881e6b5a677d973d047ad14c099c06f0941
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 3.416746947803346, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04915567383423248}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.41219549454005344, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019884907431696104}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.49154498554014486, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018555667598375688}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.43965168599123006, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0015705219676545596}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.17378317584242825, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010901893538702665}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.21285050587435922, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001428926493666674}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.18707996026648183, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010713647179164203}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.3073557477422296, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0011789635478387738}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.37811034759792467, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019549694924619787}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.33221821033645077, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0011795568835039534}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.33947329485158056, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015891394303630454}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.40971450190173514, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0018376040861944267}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.3638009261252252, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013481348026921308}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..00a6f65179d93eb8bb954c8ce0c7129cc4279d5c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 3.8370970182392985, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1144591225447721}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.33744773531264344, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002678432208002564}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.2779081994219456, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002272364632457066}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.2885492968351212, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020292469178824456}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.08918695736438806, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017352721360830848}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.07226568712760938, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013494607128344402}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.0752688547278534, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013552863861086071}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.24853209420474298, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002175861850486939}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.20284289627233723, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017347865605118378}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.21099118017935053, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015717685572481305}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.2808597038267283, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0024612171563718217}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.2298552776553825, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019962013821171482}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.23909437838411668, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018320148819675503}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..72d74d8529ae0c61bf52389819a8a7eda7663183
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 4.373718657948969, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11887879947848155}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.3522552603743486, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002763858033996764}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.2889714759845472, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00228492884684045}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.29973019758970454, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020502121064942818}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.09934840296950617, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018189448808929704}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.07999739923020184, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014210909684714815}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.0831703337253987, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014123449794312788}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.26226475143890166, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002301455268301791}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.21294988420936256, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017815097701725465}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.22110973690939364, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016159023437830125}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.2946849915708113, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0025524376570338707}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.24032554668210832, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002050233303327457}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.24955436884636814, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018809736592006266}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec3363a0404e948ee420a99c92e265e665624371
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 4.872872536054477, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.17994394529095062}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.360735661960322, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002835390448997847}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.2944152190187286, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023207814101633794}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.3055513704733046, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020673466657850498}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.10920157678214337, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0019434279841216329}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.0870860469506747, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014872544127445687}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.09057857385566902, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014742462627991503}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.27244358337055397, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0024054386143698534}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.22030127184406081, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018506185371882098}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.2287609415460662, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016617607367168975}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.30429745199273023, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002612593595236828}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.2470915077136621, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002085484889135651}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.25668504598663233, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018889983272758794}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..dadefdecb9499ded8fc9153145a84d852aa32e01
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 5.212454694507268, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1288163730607949}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.36456574097223493, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002845913293926449}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.2989078546481851, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0023201439477763606}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.3090786443943064, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020670761005209533}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.11386196869648488, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020442207769660018}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.09115599764568338, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015501355013158944}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.0944527466973476, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001527049255606511}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.2769682674664966, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002470609873235701}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.22450967972348715, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018729810994751136}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.2323973425908195, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016905329597190666}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.3089928251978803, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002660795958922366}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.2515644789803418, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002098522007442167}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.26051491605603, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019072972536883094}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..612e95e407a828ac03e779ff3e0b7ce1cb977d83
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "bleu": 5.326249606769673, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13870766757762862}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_precision": 0.3689871087097193, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0029017082283349553}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_recall": 0.3001344146565885, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022905081784066715}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge1_fmeasure": 0.31118667832544594, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020471931429837612}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_precision": 0.12022586215655875, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020979416142716933}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_recall": 0.09432946255117527, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015316319038071217}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rouge2_fmeasure": 0.0981873842191505, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015253172592372794}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_precision": 0.2833630391152307, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0025014801176794616}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_recall": 0.2280006358827552, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018282950889621254}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeL_fmeasure": 0.23662330266166123, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001654720258750121}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_precision": 0.31356761141424766, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002691111456370736}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_recall": 0.25378328346517437, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002063376374536576}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "text", "rougeLsum_fmeasure": 0.2632319188900435, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "cdecbb5a-d3e8-46f3-9ea8-22025bc59e3b", "prompt_jinja": "Information: {{meaning_representation}}\\nDescription: ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001875582937147226}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9d05b87e7040b0e74fce0e05694b58613fd9e580
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.07849100625603515, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0013763236409170894}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.18830692834903057, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003201584532139998}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.10930956321205572, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018636365300419618}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.0101373697747153, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005725799485965975}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.02554990695060758, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014536715782248486}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.014331752023111665, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008038622113235613}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.07153361180580091, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001183235553725598}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.171439117832265, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027508968911593348}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.09955783945841405, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015927736901503137}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.06237679092272722, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0010796552208218652}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.1514930768097993, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026421209389933848}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.08714871620375533, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0014790422117723686}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.47457826897610184, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06622527858136913}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..91f0601a865b132424c9c22b2ed1513d339e14f6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.09609231578553688, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015668691591351292}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.23552842368238433, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003577814966154032}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.13474307731979848, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002111986178677571}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.0110354532279438, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006463903956640263}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.027990471006616467, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001658813041921148}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.01563559121007456, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009139886175241735}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.0734302137042163, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001138075001700217}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.18134020183601798, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026912992930091363}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.10311323732086543, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015375375103156618}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.07630876213778368, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012122651784603749}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.18889813705308528, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0029100234368416634}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.10726294947450196, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016523566097021156}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.6132540544643678, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0713885076138697}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..71653790b8bda9604b071216ef6ba26211657c83
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.09383796110256946, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014865897795411922}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.2300444944691374, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003392043854938688}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.1313944002337063, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001990658173774306}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.01098830723574462, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006303776417170931}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.027489027406269212, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015570004325674606}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.01551951461927907, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008825861261678188}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.07370182535719132, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001100110897417761}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.1817594002432442, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002569209339964884}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.10329061631237338, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0014671878058074702}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.07447014880573084, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011974801492887503}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.18408290764630678, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002832630645866158}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.10446611375407802, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016139685037892917}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.6167432807079934, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07313238207352012}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..a8016c64a21bad931cbbe377d35765ccafa7070d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.08929232207008256, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017661136394464746}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.2083714353056586, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037507822542893463}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.12223241280960181, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022597737898330176}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.010779714787610186, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007228389047199584}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.024684029325778328, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015436289252720226}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.014615300016460326, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009436727425446628}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.07198363558556994, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013768675700732372}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.168224053314788, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002856247526588458}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.09843060072407699, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017199014486665768}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.07122439300395828, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014339663669546733}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.16714051967493118, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00304885339797809}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.097565135698147, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001816709165840859}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.6537439527112324, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08789003420216455}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..dd61cd9313888e5d4ee53ce52d7a6b25557af5f5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.027438753007734348, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020209855248670377}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.047135286894855855, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003093201466426986}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.030623951982172243, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019723679586947157}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.0036391643178718376, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000651422116785187}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.007018854013603984, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00101986519717702}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.004331423800367673, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006498030956306457}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.022758262766782358, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0016467261403303932}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.03973720943624368, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002596893758975351}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.02559588917107183, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016311116927956652}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.02236476517938642, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001641725788842373}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.03877314043404623, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0025919403907963476}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.025026908052136167, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001623239492581538}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.38568323827085005, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13140258824575432}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..02d16ed77798e6adef7f35fc80fb4281763d6098
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_precision": 0.004288164665523156, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019144304746682786}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_recall": 0.00019598446840121887, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 8.942783707027923e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge1_fmeasure": 0.0003741452934927028, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00017038377957226348}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_precision": 0.0, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_recall": 0.0, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rouge2_fmeasure": 0.0, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_precision": 0.004288164665523156, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019144304746682786}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_recall": 0.00019598446840121887, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 8.942783707027923e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeL_fmeasure": 0.0003741452934927028, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00017038377957226348}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_precision": 0.004288164665523156, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019144304746682786}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_recall": 0.00019598446840121887, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 8.942783707027923e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "rougeLsum_fmeasure": 0.0003741452934927028, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00017038377957226348}, {"task_name": "gem_xsum", "prompt_name": "DOC_boils_down_to_simple_idea_that", "bleu": 0.0, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "2b9c75ca-2848-4a63-b3ce-b86ea2e2d7e8", "prompt_jinja": "{{document}}\nThis boils down to the simple idea that ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..31de5d8393bae0c573dba4376e0ecd7eef51a746
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.07695974919266163, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0014773165827520049}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.18662101528755234, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0034322790231989847}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.10741115659863423, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020029906467731195}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.01122477293725975, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006155567811512096}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.028017590709361054, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015078040578957851}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.015793527267809353, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008540265285033742}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.06992993909202184, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012721116443108543}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.16999666228834834, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029761829036210304}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.09765196595353162, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017221112550801897}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.061013847977830764, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011300630372763191}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.14954259216443205, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027335378592026204}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.08537020359960786, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015426696504085384}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 0.5835098927919238, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07719313039217032}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6012fedcd575bc40a4496303845a1d3e4741e138
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.0998197720955355, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001791958979970089}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.19150976792164776, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003597233562430454}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.12292814399550107, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002063269026074902}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.00926872748126837, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006094283518785105}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.020793570369667533, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014099011327662694}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.012260258344477375, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008029144977571671}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.08018556273037858, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013905565313662618}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.1534507156318649, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002735628237761709}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.09847662810007793, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015558437843189248}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.08017202493423412, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014158245846160884}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.1538977801609464, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00287738921463875}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.09851599027450329, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001607691573132454}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 0.5393663863983892, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05348433562375772}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d5a32ce6d856b4206a349649203ba61bf1e9125
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.09588515299897513, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018389413668098858}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.1941351180627413, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0032321812427151307}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.12080327979692661, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019325844096013765}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.00963799324260621, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006615867592044398}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.020245217917730207, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0011797301011867512}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.012201390939302973, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0007193995074256504}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.08057175111690566, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015714221610920258}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.1632996891477624, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026283320276119515}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.10138824592737848, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015778531626384088}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.07734192765900497, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015709846641160572}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.1560636734165814, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002615177895498355}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.09694968298249196, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015608436285872904}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 0.4248411619556655, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04337063842371632}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b501f01db42b761d61a873c45c6ce33c62a28f53
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.0979524500570362, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023574213995995338}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.18432267940454714, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003449394808563609}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.11838286391586506, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022575984998313976}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.01223431053167332, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009139707393864935}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.023333668667089247, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001381718229050345}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.014849672103797189, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009398723971147706}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.08382542142444173, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001948560716573151}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.15921404653049828, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002868932069201499}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.1015835231933467, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0018425036048339849}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.0796692729044991, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001955663224730415}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.14966156017894405, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028188651661202858}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.09581512027677436, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018225664745385676}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 0.5384672086268061, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05628261735256145}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..cea4a0e80bb60fa593e50a7c9f446cfed08033f9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.03078887567599419, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002267198013309019}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.04728638198757095, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0029563776305562995}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.03268274482045124, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020159690172151102}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.004097645217219916, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006352569090777523}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.0070103107833449875, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0010527888125515437}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.004600261742688949, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0006815109118145586}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.02580787661904382, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017950648224843338}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.04086841692598574, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025615977979740897}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.027882226495475622, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017012154601545335}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.025250388596127913, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017916684649173303}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.0392238241311461, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024646800606543846}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.02702176662269746, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016712710261589593}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 0.30242530199415063, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13172883284690357}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec5c2a8ecb397192e093a9df1d82c1500858fa69
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_precision": 0.0024946317337081986, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007459407337053418}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_recall": 0.0020432834635293762, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006122723724231832}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge1_fmeasure": 0.0021779269831245112, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000636647151495716}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_precision": 0.00015842386125404991, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 9.164096613628691e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_recall": 0.0001078167115902965, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 6.282744363562356e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rouge2_fmeasure": 0.0001270035372229107, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 7.339490340552063e-05}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_precision": 0.002177974195907487, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006242303095573485}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_recall": 0.0017464607065091375, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.000498369445585121}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeL_fmeasure": 0.001888812253805427, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005328781795731039}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_precision": 0.002231576254226526, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006354831460676118}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_recall": 0.0018536648231472165, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005527691709280919}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "rougeLsum_fmeasure": 0.0019602816648974794, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005561333735819992}, {"task_name": "gem_xsum", "prompt_name": "DOC_tldr", "bleu": 7.569559051730558e-44, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "db54e9b5-8ca9-4266-a773-695a3dc5bbf4", "prompt_jinja": "{{document}}\n\nTL;DR: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 2.3751737130499017e-36}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a9753b16cd979bd21c728102d9fc623392e6cf3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08684854788612234, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016136017598306646}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.21068678976862762, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037297139710485957}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12137792760871402, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002184184971255157}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.012636632651205262, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006618849333669708}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0316888941525413, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016796888756788677}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01783873426577586, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009275277075872498}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07624380858300416, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013077167367173933}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18587184963214815, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0030939829912028922}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10669020751320313, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017737444033652681}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06773047218424798, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012398806155871941}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.16635584373488718, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0030019144404050606}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09494249315705383, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016928172404188113}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6141824165767692, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07244482599281088}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfbbb47c1dddfaf93d4e094996234b4e76010bae
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09856438801955326, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017913961749217754}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.23807395731457542, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.00408145979752075}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.13729478048942775, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0024098935839123887}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.014805002932563531, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0008137316866053345}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03704847337143438, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0019513225466723323}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.02086687296490721, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0011212733091223604}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07886562535884908, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001319440885298338}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.19199210330518746, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0030607720042053164}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.11005303698684504, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017730014084980385}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07731479241226244, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014429529345158431}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.18826403944987077, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0033888132406159205}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.10790506830750389, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019575871972268514}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.8152552954236955, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.1031507916283277}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..38c94452bd2b1648a8f010ea38a12566916000bb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08892601657513456, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016077156412850602}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.21529785402215762, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035920176085112838}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12396034301655152, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021408486311541627}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011265761012393422, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006031851124179686}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.0276901962071342, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014341330838498428}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015790474040565182, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000831397853071339}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07425540521394834, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.00121183146821238}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18108047541087044, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027630960930183058}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10371163259608823, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016126497685603005}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06968809353220383, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012730861197946873}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.17037941440049567, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0029297652528551157}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09738815767936584, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017074564850241471}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.4783982759526829, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.041614834308327325}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..87b748bdf3a00c94e41577b17540690df3a5ee12
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08435225307357676, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017949929630033349}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.19524716118009153, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003729643037639876}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11451534571985687, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002245243404997308}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.010931147147210569, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000631905230395927}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.026021044054339443, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014337475560734076}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.015065187296172565, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008474739343157203}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07171273980778682, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014130706113843967}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.167240338703402, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029869789558385}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.09755266669239072, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017557981664671374}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06667665265151085, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014209048598926293}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.15545938043826846, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002970962161003125}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09051613592037677, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017384555593769912}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5544562884765827, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.03868981500279069}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ae45689a3339e8cd79b257a1b8ccc21ecd2483a0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.028606404294185678, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020371981833308224}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.05177166241618001, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003118570625917115}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.033226658072660585, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002003945138254869}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.004053086557676976, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000644383420856288}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.007133537400455768, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007785336270850277}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.004530040031767069, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005161711108430996}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.02472463147760117, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017881996232063642}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.04459955593168214, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026285803260540815}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.028498538822695166, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016689936239707673}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.023409670499661615, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017830228948821282}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.041408985247042875, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024949239488166234}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.026611692302705815, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016132169063633166}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.23195231943321903, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.054461911286189185}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..fcac3cf5ed05256b1c61862d8301f302d2cf86f7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.002544888964583456, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0006804984372875271}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.002139575841226647, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0005562502135583049}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.002279670952470671, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0005995355838360033}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.000239072186907925, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00014515151101602653}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00019053273770254902, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00011332476106214262}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.00021112542779058987, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00012666146238130252}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.002161847481010399, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0005749806010887603}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0018323415576938233, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0004753823971115864}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0019409469526661608, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005073885489964555}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.0022997709059248868, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006186158865403682}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0019535380516075003, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005119627013983172}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.0020687632617786135, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005471176517107834}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 2.4598586849321513e-37, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 4.3357176079980746e-32}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ebf632927a4f527f95fd3f5ff2623cda078012a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.08294449475355638, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001501101677358941}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.20318981866441785, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003451770742452455}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.11615011773689751, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020132702860050805}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.0115654770629999, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000620636009223327}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.029012948159446418, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0015609513028227002}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.01634264454041957, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008732318540767917}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.07485979068030396, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012592589840183614}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.18415577121037147, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029429725468367554}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.10497029887714841, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016965946158159747}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.0654649026019978, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011528108608884256}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.16188192696498585, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0027459266075334754}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.091932294749694, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015639138169942146}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 0.5173175787874577, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.055369098931755845}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..63e62399e76ce74a3259cd4fe41b0549c65466f4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.08748874292219283, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016046785625036608}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.21040419098884006, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003596032405414458}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.12132677528862063, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0021172474639601026}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.012177718487699685, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006600449164619634}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.03041617462460392, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016279180469739754}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.017118068362416517, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00091598354274057}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.07463066246768363, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012825612167355926}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.18070029655502712, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029520412330890744}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.10363956196342357, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016890572561683093}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.06928118345477047, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012737323592854019}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.16792413773506376, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0029340056636205994}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.0961985987954344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016792470214151791}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 0.617354720838304, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.06307890231427365}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..19b1bbab010715b01fc32caaa2a42e26a42e8c8b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.08502500988635123, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001498937813320831}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.2084492768665317, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033474975083069305}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.11881862682601178, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019599355384508203}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.010957922059575794, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006364547528005359}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.02745994543864676, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014679599804070377}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.015337369323206902, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008247870081098087}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.0739732762619997, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012237819610442023}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.18227843033716665, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0027816909692955204}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.10352955674946399, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016013916021151678}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.06740414152435432, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0011978138296961046}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.16720075001064724, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002795799505244066}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.09449014264304802, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001581094844660653}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 0.528150461312332, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04887622861962553}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..84092aecaa59f684ba5e5c2fb78a13fbdd4d041e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.08286382508828975, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.00171211565369596}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.19417127366399595, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035347086651873116}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.11303815097050755, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002085394908569434}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.010709281907276956, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006463393919380488}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.02563055239095909, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0013753043663831622}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.014667451968252554, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008103234054454378}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.0723252465078549, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014008899550456907}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.17025301778061233, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002944125742781433}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.09884209508770671, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017207139349791946}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.0650622060288472, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013556680905752983}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.1537510648011344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028634633132608622}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.0888642550425071, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016417562882641062}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 0.4932998587527608, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04917973267537893}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb27bd917de4f7c28113f45e8d8dd39911758252
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.0299789202555777, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0023310463175663133}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.05033299021370721, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0030688602223686704}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.032765818131531135, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019956975528504003}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.003955464420202306, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005645114147565559}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.007608591554539119, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008849021378583971}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.004765891573358166, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005748770709346177}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.02585211610210547, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0019823318353263796}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.04414951473810684, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0026739955507568457}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.028427633970046932, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001695433517325568}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.024288617878882954, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019683052842310715}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.04024052293200098, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0024657230773210245}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.026135699640989368, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015948963052667224}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 0.3152874467334013, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.07390349309399162}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..e375ced474a6d8347b540734e3cd8eb124d03d5a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_precision": 0.003145834181525519, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0008641359171585344}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_recall": 0.0027491540556537565, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0007447118682849743}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge1_fmeasure": 0.002850102432403856, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0007630282877736268}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_precision": 0.0006187008822968868, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00029194397834022607}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_recall": 0.0004933724831937344, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00023534159735572293}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rouge2_fmeasure": 0.000538565784940234, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0002527457661965069}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_precision": 0.0023242531040883747, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006266425117623151}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_recall": 0.0020438981820361424, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0005399846248395976}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeL_fmeasure": 0.00211639976529527, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005548761683122237}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_precision": 0.002621636659839826, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0007128110133984216}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_recall": 0.0023514083301448595, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0006536022122755625}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "rougeLsum_fmeasure": 0.0024026672428148917, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006455426387014536}, {"task_name": "gem_xsum", "prompt_name": "summarize_DOC", "bleu": 8.230901975344253e-37, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "019726f2-7140-4ab6-a18d-a5f9cc709a47", "prompt_jinja": "Summarize: {{document}} Summary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.6818144591792003e-31}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..545966c434c7deac1460d6ffacaa1981b3650e13
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.08101405207575763, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015402709854789834}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.1978371464700832, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0034969455098272624}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.11344123075110753, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002075752711472246}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.01181630825554818, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007223475673028841}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 0.029271962971207783, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016894106447306921}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.016637993147190867, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009926063504469168}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.07385599916218046, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013240985175350677}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.1808620333503495, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0030065019141283108}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.10349719856532423, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00178021572550937}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.06323384533522881, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012031428852023567}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.1564882734228425, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028531783790494997}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.08884476245308559, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016345699043350426}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.6162431159385546, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0719411252845223}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..856586cc8c00df67f496623438721a538917fee3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.09212077005868066, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017630634952107368}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.21369522061211693, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003729436274385912}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.12537834249633983, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022469691506953465}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.01206376214517874, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007516662252398531}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 0.02903754360252403, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0017031825569670093}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.01660477741264796, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010056200276438794}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.07655663031331215, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013855875426255315}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.1792751829195347, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029860001198346964}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.10446236594583992, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017570999718229995}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.07232336786129806, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.00140309509381785}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.1689005605679794, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0030087684057466145}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.09853403043110821, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017820950507240858}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.6233103019291443, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10322680839865134}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..4cc289af3b473df29bc0814950721568f51620c9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.08661269663304301, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001593838459943478}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.20717931900250192, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0033843575265547453}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.11974042458990046, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002030784483754028}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.011007610065072464, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005989306753914806}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 0.027125371681868536, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014187760277359451}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.015360819588816904, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000809551101724943}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.07395213433806418, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012575784598958418}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.17867696254495863, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002758667555218375}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.10256512747901998, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001605784979744158}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.06701000937643628, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0012330207864759619}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.16211217586867194, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0026761095348479098}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.09286761848139825, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001559907403116052}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.4996183843525882, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.04001578639941148}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7121e23494c90b8ca4fd1af4939b00c867bfc175
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.0806895748479458, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017148726001275283}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.18907123674289394, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0036401681772858063}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.11040951377527043, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002179873874618349}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.01053868364581689, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006210803937322794}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 0.025668261986885543, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014569341217374899}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.014649332777625616, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0008477225467246834}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.07010786798662083, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013932486949560677}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.16531909956669813, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0030227929410000416}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.09609527909787907, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001772609518829833}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.06390479614025571, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013316010773909046}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.15100544107920313, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0028997429382766494}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.08755892275631494, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016919775746021333}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.5195966300461277, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.042459426003985785}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d22dbc55b04923ffe9e19b9ddcc3ccbe4130d60b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.02658254011537247, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020748261577405607}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.04377731709316155, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002784901327388771}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.029279070120697838, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001878769664661074}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.0044766254520966255, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010428981904857266}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 0.005945068157009263, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007338081846565158}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.004119906223301282, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005578664016300015}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.022618399699937414, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017607024402004165}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.03769542551971095, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002353427232761915}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.02506826654645877, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015851273432252462}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.021501210074057614, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017408075966930808}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.034957744191750476, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002217068454307888}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.023443260455568726, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015176085253178068}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.21210378195277477, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.05267300095484724}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..95c0bc052aebd17581e4b117a26c4547f9a3fbb0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_precision": 0.005288736420811892, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018191165622398336}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_recall": 0.0006712474238977358, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.000231168077080196}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge1_fmeasure": 0.0011581184102891899, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0003946303405826163}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_precision": 0.0008576329331046312, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006061777330437759}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_recall": 7.576832303743922e-05, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 5.452796435684482e-05}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rouge2_fmeasure": 0.00013885485583598793, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 9.966211058781627e-05}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_precision": 0.005002858776443681, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017037398901424319}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_recall": 0.0006283657772425043, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00021043869960245262}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeL_fmeasure": 0.001083541633497483, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.00035783272085016286}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_precision": 0.005002858776443681, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017037398901424319}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_recall": 0.0006283657772425043, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.00021043869960245262}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "rougeLsum_fmeasure": 0.001083541633497483, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.00035783272085016286}, {"task_name": "gem_xsum", "prompt_name": "summarize_this_DOC_summary", "bleu": 0.0, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "7d3584c5-8864-4d11-bce9-65499cdef4cb", "prompt_jinja": "Summarize this document: {{document}}\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..281a01b6b1b9b844be7bdd54742df548e67c8c61
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 3.656755366303797, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.14837890695680994}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.05669470574943534, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0020170312039152695}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.5066296296346544, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.007659513774926616}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.09106715079841605, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.002391471928167312}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.03860274611467539, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.00172983035557276}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.34854187631620626, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.00814423352083145}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.06256105007088962, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0022953455816733365}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.05563894642256185, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.001982960805433701}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.5011913179283217, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.007700971199463536}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.08960277978128786, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0023665012918680627}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.05312130935399373, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0019781535292318657}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.4801456290593825, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.007794216102275114}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.08521160683074443, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0023490297225075376}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d6cc361b74103170a1f70b149e59bd20ea5fb3e8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 0.8830608958021205, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.08562321028264006}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.03607709470611491, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0017441028278732172}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.1952470751009191, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.005487103316834906}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.040647935659745706, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0014550077414889832}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.007972127932130054, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0008580787510736185}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.07120120024573734, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.004600441634483666}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.012117898396108417, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0010508174756117949}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.033979273307145695, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0016175369455391048}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.19294098495627118, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.005472126990576816}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.03909307269627734, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0013947783444551163}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.03227030109817833, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0016612299817095036}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.17402891178983584, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.0053066479466079685}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.03550984850410223, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.001363844016926205}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..095600fbaf6dc92d4ff63a784b42ed3c991306d5
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 1.4935006686116292, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.05443694683302151}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.05267853270925152, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.002449745789546971}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.24860214672787803, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.006500686399418739}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.06032529701781306, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0022263011442644493}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.016806908973702062, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0016553589028564524}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.11666497774928083, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.005908602168091971}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.023433042203883557, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0018005053734734208}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.04925134882919058, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0023033509345347773}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.24482036799051923, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.006496173955774258}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.057693738636468685, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0021627311467575755}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.04806464483998574, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0023691482012047245}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.2255921129994038, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.006372394561834973}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.05423256001424035, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0021531740526494}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d03b85baaf38191ede190185eddb45c938fa9475
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 1.8212244525264603, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.09849562818662225}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.06508938178602254, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.002888335161689235}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.26659804850838303, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.006676058649232921}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.0723460691197474, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0026163497556041058}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.023241183787932676, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0021132241633176545}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.13124281702649712, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.006161317732310535}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.029903412643591754, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0021799636798326538}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.05994865949151067, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0026955181247782334}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.26144434776062075, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.006677302291405039}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.06828781879330677, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.002518729485584185}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.059049668114767744, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0027649257923785986}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.24459992364434763, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.006585924906880039}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.06528793058605904, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.002529000694923454}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..cdc17f9f2ecb52428f191660670b064b362331e6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 2.0235091580656346, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.06992480281711495}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.0750764899008003, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.003250878714640169}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.28071950983923893, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.00668526364483302}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.082090911255068, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.002803446342789559}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.029562846153838467, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.002443964528445172}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.14150750671815635, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.0062328605351622945}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.03508516074414074, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0022612345217735254}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.06986898167803665, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.003066369740945722}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.2749317176165797, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.006684573146158844}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.07771533962795445, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.002693141545803596}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.0688802573137848, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0031419995471746284}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.2583791181007065, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.006608286981904223}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.07481316087741567, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0027185549354669615}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..32838f371e5006d102316b2207ce1910c0f25fb3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_Correct-the-solution_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "Correct the solution", "bleu": 1.9681135945636459, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.13208724741948602}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_precision": 0.08049093429476127, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0035142461324920273}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_recall": 0.27883210803766567, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.006581683130296607}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge1_fmeasure": 0.08562740019962164, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0029691098655082715}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_precision": 0.0333369150257513, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0027845366791002944}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_recall": 0.13700194089143813, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.00613423127185248}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rouge2_fmeasure": 0.03697008601032397, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.002496780466003745}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_precision": 0.07459959278412087, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0033268926264176904}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_recall": 0.272528207623618, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.006590906751190863}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeL_fmeasure": 0.08076583460906837, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0028795378570412307}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_precision": 0.0735499511307004, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0033973926098363675}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_recall": 0.2546069662211679, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.006496163637745307}, {"task_name": "piqa", "prompt_name": "Correct the solution", "rougeLsum_fmeasure": 0.07760507931200265, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "5f4b4645-9438-4375-9062-083130e6d04e", "prompt_jinja": "Given a goal and a wrong solution, rewrite it to give a correct solution.\nGoal: {{goal}} \nSolution: {{[sol1, sol2][1 - label]}}\nCorrected solution:\n|||\n{{[sol1, sol2][label]}}\n", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.002889469047971454}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3d63643df108ff5bb61fd82b44feebd1f96d5c2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5048966267682263, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01166526473007815}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5048966267682263, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01166526473007815}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..71232b23c3357f5fa70746dd3f4401e6f949fc43
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5016322089227421, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665762007194868}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5016322089227421, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665762007194868}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d13e3c284e9eb4e6f3fdd08292a37d92ba11a44f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5048966267682263, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665264730078145}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5048966267682263, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665264730078145}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..110f4d3d7c925571f31ebed1047861855ee9e97d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5272034820457019, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011648545262429021}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5272034820457019, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011648545262429021}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..1cd4fe73b3e6937dcd844fdda39f02ac2974f311
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5125136017410229, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011662170084916898}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5125136017410229, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011662170084916898}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ef091b8fd0d8b5e34001c4646b3cce8af915ff00
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc": 0.5130576713819369, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01166184537588635}, {"task_name": "piqa", "prompt_name": "choose the most appropriate solution", "acc_norm": 0.5130576713819369, "fixed_answer_choice_list": ["Solution 1", "Solution 2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "adfef248-f856-44fa-affd-e3223718854e", "prompt_jinja": "Given a goal and 2 solutions, choose the most appropriate solution.\nGoal: {{goal}}\n- {{\"Solution 1\"}}: {{sol1}}\n- {{\"Solution 2\"}}: {{sol2}}\n\nAnswer by returning either {{\"Solution 1\"}} or {{\"Solution 2\"}}\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01166184537588635}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ce39e5532ecde90d0e974ff0e8e888875dd2fa8f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.06614185025884788, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.006992904774416279}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.016184382076826428, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0008939401678502145}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.14173571994695022, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.0030334268104372647}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.024878431671277717, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0007084095842894381}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.0014202808708619317, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.00013696486457806}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.014499650869656043, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.0010689452794773893}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.0024039653739287383, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.00019957865940879526}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.015195809153249718, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0008191061120771218}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.13579609932870865, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.0029357166668511343}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.023587495220817104, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0006681752509678566}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.012906484148544855, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0006299772819920492}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.12046291801697726, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.002609561210140041}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.020150738338231722, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005489881127507307}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d49fb71eb11cb40f5f5d345cb76b2178af65755
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.10718681853330272, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.018000911234593346}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.04132674547577212, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0024485259308753586}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.08689740514715649, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.002916092372055045}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.03703813399564072, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0018091336796340791}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.0071242151063781395, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0008842251520965283}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.011318163060566497, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.001006748832876741}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.005941822792449927, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0006087286220180097}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.03625394677542022, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0021263148272352293}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.08136452827721419, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.00276242885759181}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.03303428861037457, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0015956649904201666}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.03652614662967736, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0022174820770640377}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.07577016170577146, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.0026023305739597473}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.03218887137036506, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016050372004655393}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5cf92163c084a4f933d79bce743faca325d47dcc
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.1712273909909647, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.029845744864029765}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.045080131535418684, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.002665199214443293}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.051600544061536885, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.002562420942904496}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.03482910285332883, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0018619215722441118}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.006360924839859195, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0008787433792928841}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.007684278380676342, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.000961471039023744}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.005293218483445248, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0006663810940536303}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.0394285879549089, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.002356887213890825}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.046948342759142385, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.0023931560766638333}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.030614463329020847, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0016647185279519317}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.04033700739178538, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.002458798837565943}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.04587514391567024, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.002316207868651937}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.030839298564677823, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016870567056580634}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b950b902a3149cb2b004e0e122860509cfcb4256
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.22910425239933835, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.023846299745637276}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.04471001776599436, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0026879796556863317}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.045685551243527515, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.002440523184391023}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.034617382780722655, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.001904100064118123}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.006631747554370845, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0009077372437363107}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.00708411646538401, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.0009178652859708077}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.005307127222312835, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0006509675188338064}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.03921165897731945, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.0023674286187342075}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.04101390938445623, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.0022350756769902065}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.03053639049565289, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0017070590062506794}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.039886138935695335, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.002448586484801831}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.04031563564195608, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.0022007717563761603}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.030622384698934416, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.001718163099004444}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd77d197f53e2c2119ac9d53a9b78257fc233e0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.20107349281952527, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.04214224677468119}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.049366458127839984, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.002854014732476892}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.04923533037170826, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.0024982212236816293}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.038633264035856554, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0020222776724184797}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.0069147124408860524, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0009836165108404464}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.0069063816404413315, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.0008881448140462662}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.005146040515208341, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0006133922293712356}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.043511782572989245, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.002558689962342701}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.04427204057886875, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.002279980453794631}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.0341478013156332, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0018057433669060028}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.04448265410501512, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.0026386860903847256}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.04405637966133121, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.0022677196418328066}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.03455201836586164, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018292045551711296}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..f02414acda6908589a97f275484a22d12d72ae33
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_no-prompt-needed_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "no prompt needed", "bleu": 0.2315660064365379, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "bleu_stderr": 0.03985185692641373}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_precision": 0.04883644295164218, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_precision_stderr": 0.0027824290368789364}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_recall": 0.046722404599889716, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_recall_stderr": 0.0023869495808978747}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge1_fmeasure": 0.03797730190196727, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge1_fmeasure_stderr": 0.0019584581823532746}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_precision": 0.006695738235568391, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_precision_stderr": 0.0009608876674204776}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_recall": 0.006591972710456567, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_recall_stderr": 0.0008806024888411368}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rouge2_fmeasure": 0.005234740517822527, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rouge2_fmeasure_stderr": 0.0006778104688310928}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_precision": 0.04343865379951131, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_precision_stderr": 0.002509403663109993}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_recall": 0.04203034225897826, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_recall_stderr": 0.0021838302966321445}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeL_fmeasure": 0.0337124518984988, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeL_fmeasure_stderr": 0.0017510459198918948}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_precision": 0.04410239398251147, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_precision_stderr": 0.002562032316728619}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_recall": 0.04154031189118946, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_recall_stderr": 0.002165265517853104}, {"task_name": "piqa", "prompt_name": "no prompt needed", "rougeLsum_fmeasure": 0.033884080328243846, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "c8c45ef1-2ffc-43d7-8710-b98c2fc4f699", "prompt_jinja": "{{goal}}\n|||\n{{[sol1[0].lower() + sol1[1:], sol2[0].lower() + sol2[1:]][label]}}", "prompt_original_task": false, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017631801582756708}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..016671d9c2dbfa68ebcd06587adfa0a2a68637ef
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.49510337323177367, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01166526473007815}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.49510337323177367, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01166526473007815}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..619cb2ff7e99a2e7ec98ba41ffa2647531dce64c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.499455930359086, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665817258899168}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.499455930359086, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665817258899168}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..930b2c2623e5fdd2ff0ba144fb6e870ca983f033
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.5054406964091404, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665133500637066}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.5054406964091404, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665133500637066}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4c8548caf358c5eec9f93270ac965645485fb194
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.515233949945593, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011660408257153638}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.515233949945593, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011660408257153638}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ad1d4cfd5386df0f796b7ba3ec45ee5217a0916
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.514145810663765, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011661154475524836}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.514145810663765, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011661154475524836}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2d9efe9d1723f2082d6389d79385838163e728f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc": 0.5087051142546246, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011664055982032842}, {"task_name": "piqa", "prompt_name": "pick_correct_choice_index", "acc_norm": 0.5087051142546246, "fixed_answer_choice_list": ["1", "2"], "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "44778818-7b73-4262-a294-c00fc32b6c2c", "prompt_jinja": "Sentence: {{goal}}\n\nChoice {{answer_choices[0]}}: {{sol1}}\n\nChoice {{answer_choices[1]}}: {{sol2}}\n\nWhat is the index of the correct choice for ending for the sentence?\n\nAnswer:\n\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011664055982032842}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfabb503f419fcc22e43f6df661f13aa13f1101c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.4956474428726877, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665382144642394}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.4967355821545158, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01166557553076037}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b4cb7e38390f16332306fa616f312834e39acf4a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.5092491838955386, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011663828032649187}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.5092491838955386, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011663828032649188}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..7053f3b676ab01536558dea22e7b923cb49a4c3c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.5108813928182807, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011663061261117737}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.5048966267682263, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665264730078145}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..6bb89686dde6d36ac20b0aca4c8b24fd4b1a3954
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.515233949945593, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011660408257153638}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.5070729053318824, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011664656918145945}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5fdc3fdb2b31d5b330a35f64dda7aa5f47fbada
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.5114254624591947, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011662778026451675}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.501088139281828, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665796539540878}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..62c7b3f41ea3730779f05cca39560cb92f43e5aa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc": 0.5048966267682263, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011665264730078145}, {"task_name": "piqa", "prompt_name": "what_is_the_correct_ending", "acc_norm": 0.4972796517954298, "fixed_answer_choice_list": null, "dataset_path": "piqa", "dataset_name": null, "subset": null, "prompt_id": "16e97a16-c958-4956-bfba-279f88dafd5b", "prompt_jinja": "Goal: {{goal}}\n\nWhich is the correct ending?\n- {{sol1}}\n- {{sol2}}\n\nAnswer:\n|||\n{{answer_choices[label]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011665651503000732}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..079e6104be668df1f5505854aa71dd9a9e2e258d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.296, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014442734941575018}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.312, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014658474370509005}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f76d69e98fee70e0342f2919a56b0f412437204
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.3, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014498627873361427}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.327, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014842213153411247}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d49a96ea9c596a4075c6a37453b162c9963b4cc7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.299, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014484778521220473}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.338, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014965960710224494}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ceeec6003a83f618d5cdef319a6ec7a1cba22b09
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.301, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01451239503354315}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.321, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014770821817934637}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0ba1588b78326b850299b6ac19e83ed4d3ba6032
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.309, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01461960097720649}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.325, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014818724459095526}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a15d30cfa8fb7d328f991672eef5f887a6a905d1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc": 0.303, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014539683710535257}, {"task_name": "sciq", "prompt_name": "Direct Question (Closed Book)", "acc_norm": 0.328, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "0af52ad2-2b12-4700-b664-cd26d2da6dc1", "prompt_jinja": "Q: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014853842487270336}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6755c08a9649f21bfa02f2b23b44d1e11cee34f3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.561, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015701131345400767}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.493, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015817749561843574}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..22abd92fa3c35371f9ad4aa136f51d6b1eab6f94
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.578, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01562562511262066}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.543, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015760691590136388}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c9b40d831f5e9b1da10340ea07d26d46405ca3d3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.563, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01569322392873038}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.536, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01577824302490459}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f95de9ce72d6b340fa91855eb796911bd5ea6dd0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.567, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015676630912181334}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.543, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015760691590136384}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7b9d09bc37a6ececc6e0d4483c7e91f1e5e4cd3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.573, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01564978964446221}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.557, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015716169953204105}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a44101a7e37ba48747a6eb9141dba04b5218d373
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Direct-Question_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Direct Question", "acc": 0.573, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015649789644462214}, {"task_name": "sciq", "prompt_name": "Direct Question", "acc_norm": 0.562, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "d417fcfb-9f00-4186-95d8-e63609495164", "prompt_jinja": "Answer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01569721001969469}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..997188f64bc50d508bb5ff66a3c7f62b97a10cca
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.302, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.01452608023545955}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.316, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014709193056057135}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..145738e1e0362f7dd1a73322fa99db5af4d7f4cc
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.32, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014758652303574874}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.343, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.015019206922356953}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..4aa9d57a8a871c57c37c434e747551863b1ed463
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.33, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014876872027456729}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.35, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.015090650341444233}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..575ed98812e567c46cc0e7cff626640f179985ff
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.338, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014965960710224482}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.37, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.015275252316519357}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..baa321f74ecbcee59a30461796fc5741bb5fcc26
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.345, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.015039986742055235}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.346, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.015050266127564446}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3b71b982909ccb5fe9d44b9cd01c114581e050f7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc": 0.325, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014818724459095526}, {"task_name": "sciq", "prompt_name": "Multiple Choice (Closed Book)", "acc_norm": 0.34, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "15b0a989-84e4-4f1c-8ac1-12dbfa2ff42a", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014987482264363935}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..7d717bafa15549483d9e379f8417169e5aaf8e8e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.353, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01512017260548369}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.343, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015019206922356951}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d8cfa7caf63df5cecf7415d8c9ec5ace4075ee66
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.362, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015204840912919498}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.367, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015249378464171756}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..690759a67db3621b6bfa2b09373badb58a51e653
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.374, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015308767369006358}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.38, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015356947477797572}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..00122e3837f62d932e3c1ba91df09a6ac3bd0c96
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.38, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01535694747779757}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.398, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015486634102858917}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b67309ac1ca4ed0f143b52c1bdb9a9cadc028dcf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.37, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015275252316519366}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.394, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015459721957493382}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7dbed927963a4e04bd46e98405308ce66320220c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc": 0.375, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015316971293620996}, {"task_name": "sciq", "prompt_name": "Multiple Choice Question First", "acc_norm": 0.374, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "368e29fb-506d-4a4e-ac33-0af8d6e1729b", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nQ: {{question}}\n\n\nRead this paragraph and choose the correct option from the provided answers:\n\n{{support}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\n\nA:|||{{answer_choices[3]}}\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015308767369006363}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..8a651f611d66f12f71bc5ad085f306fd93a9ebe1
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.326, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01483050720454104}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.343, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015019206922356953}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..268db608830fa3f6a77d2e67121b9b2d01a8d01b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.329, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014865395385928357}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.34, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014987482264363937}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..bdc985672c389f43d32c884b7ce5efd8a9ca8312
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.344, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015029633724408943}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.351, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015100563798316403}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b794a6671bd2879fa7a58d094d0f39be12f713e7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.362, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015204840912919496}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.373, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015300493622922812}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e86ef69af5717ac4e1bfb642273edaa46ed26a9e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.356, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015149042659306623}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.36, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015186527932040127}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..eed3a2df8895ae5cbc49436a336ad124bb531b48
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_sciq_Multiple-Choice_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "sciq", "prompt_name": "Multiple Choice", "acc": 0.344, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_stderr": 0.015029633724408943}, {"task_name": "sciq", "prompt_name": "Multiple Choice", "acc_norm": 0.366, "fixed_answer_choice_list": null, "dataset_path": "sciq", "dataset_name": null, "subset": null, "prompt_id": "63c22e8a-7029-4ce3-bd26-6ca6a1541563", "prompt_jinja": "{% set order = [[0, 1, 2, 3], [0, 1, 3, 2], [0, 2, 1, 3], [0, 2, 3, 1], [0, 3, 1, 2], [0, 3, 2, 1],\n                             [1, 0, 2, 3], [1, 0, 3, 2], [1, 2, 0, 3], [1, 2, 3, 0], [1, 3, 0, 2], [1, 3, 2, 0],\n                             [2, 1, 0, 3], [2, 1, 0, 2], [2, 0, 1, 3], [2, 0, 3, 1], [2, 3, 1, 0], [2, 3, 0, 1],\n                             [3, 1, 2, 0], [3, 1, 0, 2], [3, 2, 1, 0], [3, 2, 0, 1], [3, 0, 1, 2], [3, 0, 2, 1]] | choice %}\nAnswer the following question given this paragraph: \n\n{{support}}\n\n\nQ: {{question}}\n\n Choices:\n\n- {{ answer_choices[order[0]] }}\n\n- {{ answer_choices[order[1]] }}\n\n- {{ answer_choices[order[2]] }}\n\n- {{ answer_choices[order[3]] }}\n\nA:|||{{answer_choices[3]}}\n\n", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.015240612726405756}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..60f132090e301cd6eb54c1a63d0e25a5e017e136
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.5056119722073757, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011561703928784335}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5125601282736505, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01155878357073797}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..97a808af94dd44821c144624bb6194ff047e7caf
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.49812934259754144, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011562351329083266}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5088188134687333, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011560633656952961}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..008c86a76e1134215bd88101a37ce77776f59401
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.4949225013361839, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011561836054238776}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5173703901656868, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011555452669106635}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7c99a27cdbeb4daafc54386ca0a05022a18b490
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.48850881881346875, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011559378273599118}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5082843399251737, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01156084507652571}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7a67df67e65578bef1f7db69877df2f0bebc85a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.4965259219668626, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011562153149168298}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5163014430785676, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01155628548452156}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..acf5e1a608eea23fd90c7ad5dd8a36c06ac8f4cb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc": 0.49331908070550506, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011561400034509398}, {"task_name": "story_cloze_2016", "prompt_name": "Answer Given options", "acc_norm": 0.5114911811865313, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a4946f9-a0e2-4fbb-aee8-b26ead2cf6b8", "prompt_jinja": "{{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What is a possible continuation for the story given the following options ? - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011559378273599126}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6e891074a8f872e07846414f6cb430114084c1f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.518439337252806, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011554566910658103}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5456974879743453, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.0115140402455835}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d40faf1e376a4b9feb5fce4a1260e8955587cae9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.5168359166221272, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011555875693960771}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5339390700160342, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01153576488164141}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ce6315124b674b4ad9f8b2111d59dcefbaa2eb06
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.51309460181721, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011558466383367176}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5392838054516301, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011526690316014594}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fdfd7751873e4edefbe6d2806a1bab4327ace25c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.5056119722073757, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01156170392878433}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5275253874933191, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011544898473864576}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a7deb59e6dc1387df7e4c0bb03baf3cfac5ba64
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.5125601282736505, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011558783570737967}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5312667022982362, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011539803085637727}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3fb93772f8e686058e7a937146304eac683fe831
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc": 0.5056119722073757, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01156170392878433}, {"task_name": "story_cloze_2016", "prompt_name": "Choose Story Ending", "acc_norm": 0.5275253874933191, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "1a9d53bc-eb77-4e7c-af6e-3d15b79d6cf1", "prompt_jinja": "Read the following story :\n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose a possible ending for the previous story from the following options: \n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011544898473864574}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..17471e5e829cc4adf85270df3bd4e80b5a29ac2b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c98ba6d5abc4408b9d972d7ff0d220588a59fdc
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b2e3f436439390d5f85dc975790620ab7475e931
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..a5561461087fc9fbba97ffad84dcddae7da8c8e7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..353855955a62c316585bf6a5afb307cd7050a0e0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3624d9228613bcda1e2e31093c8532f97ddb097a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json
@@ -0,0 +1 @@
+{"results": [], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe144b35c578c5a2c9e783c3eed98b261b6b18d8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.5008017103153394, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011562417388300206}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.5136290753607696, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011558135970599896}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..83bd48b4fae66633e538d8c3d323f1a39c48ae6e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.4863709246392304, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011558135970599896}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.5104222340994121, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011559920087347773}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..96dd79d3e6ead748e7823dac749e4a81aa9a2040
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.48583645109567075, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01155779233130167}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.5056119722073757, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011561703928784332}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c202753908bd002a308bad6a6570fd3fa4f2119
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.4836985569214324, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011556285484521565}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.49706039551042225, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01156223242154194}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d212245d3553deeb2b0f92634d16c70643f2f8aa
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.4826296098343132, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011555452669106632}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.501336183858899, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011562390964658753}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d9a2e7c2d43c6d0f6ca7223ec79b927ee8e6165e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc": 0.4890432923570283, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011559655791130734}, {"task_name": "story_cloze_2016", "prompt_name": "Novel Correct Ending", "acc_norm": 0.5056119722073757, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "c6f3d802-4f97-449f-a911-03470d418f7d", "prompt_jinja": "I read the following novel: {{input_sentence_1}} {{input_sentence_2}} {{input_sentence_3}} {{input_sentence_4}} What do you think is the most probable ending? You can choose from the following options: - {{answer_choices | join(\"\\n- \")}} ||| {{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011561703928784332}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..03264bc7e271f1b32453bb1463f2a59636010d84
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.5168359166221272, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011555875693960774}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.5376803848209514, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011529552555884568}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e9d481164c5f4c9cb9d875aa6da4364d3ea62f29
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.5120256547300909, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011559087533800689}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.5360769641902726, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01153229486915312}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..31ad81872c26c6803caa97f97235a4d65e31c5f7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.5066809192944949, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011561400034509398}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.5264564404061999, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011546234813777412}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..921d68583c5a013c3ec9ac1156df6e1b3eb6554f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.4922501336183859, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01156104327886354}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.521111704970604, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011552120807053817}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..01a606fb94c1214c63596f4085196acd6d53a4fd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.5018706574024586, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011562351329083271}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.5243185462319615, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011548748301487312}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..b679e06ade060e5093c1297e945a1a179cade2fb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc": 0.498663816141101, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.011562390964658758}, {"task_name": "story_cloze_2016", "prompt_name": "Story Continuation and Options", "acc_norm": 0.5232495991448424, "fixed_answer_choice_list": null, "dataset_path": "story_cloze", "dataset_name": "2016", "subset": null, "prompt_id": "b5c8445f-2d3a-4691-bdd5-58956816702f", "prompt_jinja": "What is a possible continuation for the following story ? \n\n{{input_sentence_1}}\n{{input_sentence_2}}\n{{input_sentence_3}}\n{{input_sentence_4}}\n\nChoose from the following options:\n- {{answer_choices | join(\"\\n- \")}}\n|||\n\n{{answer_choices[answer_right_ending -1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.011549925483927461}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..392bfbd1d8ea58fc7529a25c1eeb91452310537e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.5270758122743683, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030052303463143706}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.5018050541516246, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030096267148976626}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb5ab71b3991aa8f7b48b8200f2dedec4ac45f18
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.49458483754512633, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.49097472924187724, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..df0f480a781c7142efef5995b99363d9d0d42743
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.49458483754512633, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.5018050541516246, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030096267148976633}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a98ab20fc49504d86bb92b1a1d673c69d295d90
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.49458483754512633, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.49458483754512633, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030094698123239966}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f9a42912ca37436b8df41ff1acd0d08f21ff064e
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.48014440433212996, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0300727231673172}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.4548736462093863, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.029973636495415252}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..45c84253b2e45282418688db2abbc13de69cdc49
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc": 0.48736462093862815, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030086851767188564}, {"task_name": "superglue_rte", "prompt_name": "GPT-3 style", "acc_norm": 0.47653429602888087, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "8fb1c6aa-20e9-438c-bece-c6af1c746449", "prompt_jinja": "{{premise}}\nQuestion: {{hypothesis}} True or False? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.03006330041190266}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea7c2f7d34223ea3f9bead041a80c6ce51ac9de2
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.516245487364621, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030080573208738064}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.5270758122743683, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030052303463143706}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..eaf6e3108871ee94eae4b3a69cbe8400ec2a3af8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030091559826331334}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..bfa34f5b97e1000cbe8a504827ff836427d63b16
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.4981949458483754, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030096267148976633}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.48736462093862815, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030086851767188564}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cff24422795038d5005c3a619166eab12687e25
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.4981949458483754, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030096267148976633}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.48375451263537905, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030080573208738064}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e75a33125b7cf77eedc5716612786fcaeba7fadd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.5054151624548736, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.5054151624548736, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030094698123239966}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ecd095b02a61877686fc562844ddc4a991939aec
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc": 0.48375451263537905, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030080573208738064}, {"task_name": "superglue_rte", "prompt_name": "MNLI crowdsource", "acc_norm": 0.5018050541516246, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2b52a83c-0021-41fe-b44c-5aaa076d71a2", "prompt_jinja": "{{premise}} Using only the above description and what you know about the world, is \"{{hypothesis}}\" definitely correct? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030096267148976626}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e31a3bbd3504a9332ed0825151b7415711a02fa9
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.4368231046931408, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.029855247390314952}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.5090252707581228, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6767261b69d053a29f053e09db050b3fbe509a38
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030091559826331334}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..326eb643a03e1b695388979e9765486884305760
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.48736462093862815, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030086851767188564}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.48375451263537905, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030080573208738064}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b13eef0007decad3e00ddc410ec8a8706d0589b4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.47653429602888087, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.03006330041190266}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.4548736462093863, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.029973636495415252}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..75f041db4ac293893d8b224470cb5f9c0c5b6319
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.4729241877256318, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030052303463143706}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.49458483754512633, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030094698123239966}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c76962ecf0d1b691b5937232c3ce788d946d3138
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc": 0.4620938628158845, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030009848912529117}, {"task_name": "superglue_rte", "prompt_name": "does it follow that", "acc_norm": 0.4693140794223827, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "e2fb58f2-b1f2-4aef-b74b-c4ee1c571fff", "prompt_jinja": "Given that {{premise}} Does it follow that {{hypothesis}} Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030039730592197812}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4e075e846a35ae0c79caa620d81906c2a445db67
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.49458483754512633, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.5270758122743683, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030052303463143706}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..561b74d6ae65e56a86c2e0c1feb7820d4890d0d8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030091559826331334}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.48375451263537905, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030080573208738064}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..913efb12f0d487b6baf738b6663a034deece5224
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.47653429602888087, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.03006330041190266}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.47653429602888087, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.03006330041190266}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..06d9d542a580099d80ed00f1382e2cc1972e7a40
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.49458483754512633, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030094698123239966}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.4657039711191336, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030025579819366426}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed28f3fe2c66eb9690697417c76d145c4d4da27d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030091559826331327}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.49458483754512633, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030094698123239966}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb50ca6a4a2e457e7ea4f27580eb385e3c56c027
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc": 0.4981949458483754, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030096267148976633}, {"task_name": "superglue_rte", "prompt_name": "guaranteed true", "acc_norm": 0.4981949458483754, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "2d0d63da-ffcf-4f6e-941a-b8da922be43e", "prompt_jinja": "Given {{premise}} Is it guaranteed true that \"{{hypothesis}}\"? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030096267148976626}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9dd17c4bdac5d435b7e1f5f84ef7145201e4088d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.5270758122743683, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030052303463143706}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.5270758122743683, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030052303463143706}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a1f4d0c469a6c78ab1b86e8bbab0c9e0e07fe3a
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030091559826331334}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.49097472924187724, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b68dbf9f9c4c3b5c6df15a01471ea7b6c10a7fb7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.48014440433212996, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0300727231673172}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.48014440433212996, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.0300727231673172}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..2bfe25e0c4468adaf4e32ea67a1106a470ae1bd3
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.48375451263537905, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030080573208738064}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.49458483754512633, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030094698123239966}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..5025b5a9d1653f8d4cbad8d58b2b57c02af806a6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.48014440433212996, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030072723167317194}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.5018050541516246, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030096267148976626}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..68ec345ac4ec8504ebd9a28ce828f1aabc35f4f7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_superglue_rte_should-assume_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "superglue_rte", "prompt_name": "should assume", "acc": 0.4981949458483754, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.030096267148976633}, {"task_name": "superglue_rte", "prompt_name": "should assume", "acc_norm": 0.5090252707581228, "fixed_answer_choice_list": ["Yes", "No"], "dataset_path": "super_glue", "dataset_name": "rte", "subset": null, "prompt_id": "b8dc85c6-28b6-4340-979a-8e77c2a0dde8", "prompt_jinja": "Given {{premise}} Should we assume that \"{{hypothesis}}\" is true? Yes or no? ||| {% if label != -1 %}{{ answer_choices[label] }}{% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.030091559826331334}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4b02be22f6727b42ef88c5ff8cdc89cd3b81efbd
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.5059194948697711, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01405150083848581}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.4940805051302289, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014051500838485807}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d7d401064278b73ba9e24a8a348aa36bca1b3a4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.49329123914759276, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014051220692330349}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.500394632991318, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052481306049516}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ad663fbe4af5875bd1c7f1a32e1b37da9d1e955c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.4877663772691397, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014048278820405621}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.5090765588003157, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014050170094497704}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..103647301018d5e214589ecafb664b23b5152dcb
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.505130228887135, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014051745961790516}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.5074980268350434, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014050905521228571}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..7623ede20d66b7c1f692998d9551556c78d2d5f0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.510655090765588, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.0140492945362904}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.5035516969218626, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052131146915852}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..612efedd9cd9c0f8f0f738bd9c370e9d719f07f0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_Replace_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "Replace", "acc": 0.505130228887135, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014051745961790516}, {"task_name": "winogrande", "prompt_name": "Replace", "acc_norm": 0.4964483030781373, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "e7e42e9e-bc57-46ed-ad8a-76a5b90a5bb9", "prompt_jinja": "{{sentence}}\nReplace the _ in the above sentence with the correct option: \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052131146915867}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f4ea01a1d97e8b2416c480d7c3aaee380c893a0
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.4980268350434096, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014052376259225636}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.5082872928176796, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014050555322824192}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0dcb75ad7a0298f77bb03711865c17f9a95dbc94
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.4940805051302289, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.01405150083848581}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.494869771112865, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014051745961790516}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..53d4130689553bfaade3aaf0810f956d2868f683
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.5035516969218626, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014052131146915845}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.500394632991318, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014052481306049516}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee6f9198f34046dede40761345292c23aefb6d81
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.5059194948697711, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014051500838485807}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.5019731649565904, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014052376259225629}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ebc0d8bc6b80f0007397f673d47280673c89902c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.526440410418311, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.014032823874407224}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.5232833464877664, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.014037241309573645}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..f3aaff894384a7a1b953f438d1363c88fe703611
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_True-or-False_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "True or False", "acc": 0.5122336227308603, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_stderr": 0.01404827882040562}, {"task_name": "winogrande", "prompt_name": "True or False", "acc_norm": 0.5090765588003157, "fixed_answer_choice_list": ["True", "False"], "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "d11378d1-2f24-4509-bbbc-bfa2921300d5", "prompt_jinja": "The _ in the sentence below refers to {{option1}}. True or False?\n{{sentence}}|||\n{{answer_choices[answer|int - 1]}}", "prompt_original_task": false, "comment": "", "acc_norm_stderr": 0.0140501700944977}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..14910dda513ac2fa39f33f61fba9aa9206ba9730
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.4877663772691397, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01404827882040562}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.4846093133385951, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014045826789783665}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6660a5e8f54a618ec153199f96169c72f73c81a8
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.4877663772691397, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01404827882040562}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.49013417521704816, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014049749833367589}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..aca5512e99d3b58130621d1691592aa54801a42f
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.5201262825572218, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014041096664344327}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.4988161010260458, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052446290529019}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..cc63aa9d59eb1956327b2e365ceba6fb0667c3ab
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.5122336227308603, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01404827882040562}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.5035516969218626, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052131146915857}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..892fc7d9c125c4b4435cf49b48d05b80b0cb47e7
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.5114443567482242, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014048804199859322}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.4980268350434096, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052376259225629}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2021e9976dfc9365aa7bcea520acdcbc8383ae87
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc": 0.489344909234412, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014049294536290393}, {"task_name": "winogrande", "prompt_name": "does underscore refer to", "acc_norm": 0.49171270718232046, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "0f23f058-5b4d-42a3-92d4-5d60688aa90c", "prompt_jinja": "{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or  {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014050555322824192}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..b6a870d83540d1417510938039fbf0cc62dbc779
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.4925019731649566, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01405090552122858}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.48855564325177586, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014048804199859325}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..daa7746cabed0343be4013427d525960c137888c
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.500394632991318, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014052481306049516}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.4980268350434096, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01405237625922564}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..662d98bbb8d50e986558c15c71e47182cad3f8e6
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.5169692186266772, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01404439040161298}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.5027624309392266, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052271211616445}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..16887c2fdfc8cec86d88ddc546528ab509f78b4d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.5122336227308603, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014048278820405616}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.5035516969218626, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052131146915857}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..99e817c0a78860ccfb6b2e012dfff06897872c3b
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.5098658247829518, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014049749833367596}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.489344909234412, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014049294536290403}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7f65366edb57a5f1afbd77cd5970bfca9c99aad
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_stand-for_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "stand for", "acc": 0.5090765588003157, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014050170094497712}, {"task_name": "winogrande", "prompt_name": "stand for", "acc_norm": 0.48303078137332284, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5080f912-fac8-400f-983c-944baf9b10c0", "prompt_jinja": "In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?\n{{sentence}}|||\n{{answer_choices[answer | int - 1]}}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014044390401612972}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..663d3fec8aa17fb2ca901d49d4989a59aa00ddff
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.48855564325177586, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014048804199859329}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.4925019731649566, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.01405090552122858}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 0, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d18a1ab9a4ba507f314ad852c5aa42847da7840
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.48539857932123126, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014046492383275835}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.48697711128650356, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014047718393997663}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d2612863e172d6a98c121a6578b915a11fc720ad
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.4925019731649566, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014050905521228573}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.500394632991318, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014052481306049516}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fab6ed3d16bf7095a6b6ac5d387de5b45531fda4
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.5074980268350434, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.01405090552122858}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.4956590370955012, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.0140519560640769}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..db90d7b58564f7059a4ee4a4c0a6521e2ebc5267
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.500394632991318, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014052481306049516}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.49171270718232046, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014050555322824192}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7a0a8c71214889420761a1d4a6b40898fc9cb7d
--- /dev/null
+++ b/1b11b51b5/eval/agg.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "winogrande", "prompt_name": "underscore refer to", "acc": 0.4988161010260458, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_stderr": 0.014052446290529019}, {"task_name": "winogrande", "prompt_name": "underscore refer to", "acc_norm": 0.49013417521704816, "fixed_answer_choice_list": null, "dataset_path": "winogrande", "dataset_name": "winogrande_xl", "subset": null, "prompt_id": "5af00ba1-86e0-421b-bb97-26bf58df52d3", "prompt_jinja": "{{sentence}}\nWhat does the _ in the above sentence refer to? {{ option1 }} or {{ option2 }}? ||| {% if answer == '1' %} {{option1}} {% else %} {{ option2 }} {% endif %}", "prompt_original_task": true, "comment": "", "acc_norm_stderr": 0.014049749833367596}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1ce5f687738e3260f52999b63f57cf4d3ff8c4ad
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28640f539b985765adeb96296c17625ba2f2385fbf79dbe445c2d18ef6b3462b
+size 4002326
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..07b2a2684e7bfbdb98959b653d8aa0bc2c191f5f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fb36f47a78ddea2ac58d1565210d5da841e76fbfd9780acbac5e62a61d5ed66
+size 9643228
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fdb8b54b4e7d741e9dc0ed10f6e9bc4e52ccf832
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c287778b90a91fea90ecfaf5089ee1002deb89b20f19c5013100871b04eab728
+size 5685072
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b203e45fae436a28f267979bbe852932a368c55a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b7bcf0f45a5820fb5340dec914c6dd7491c6a2c10a2eea41d846b1d575ec080
+size 26325964
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a0e92551f7d92d444b738c11d6077990d922112d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412d26a01e86b84995b6132820c976286e91cdd4a7d74d2d45865b812152b538
+size 14956564
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7ce7045c24c847d7decaaf57d29974f4ed61d0a3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a989095ebecda92ea6505768cbbfd90fcf2f6fd271c866d7fe8abf0c11b43593
+size 8387945
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..67f399546f75dc2ae606476b08558213870c4996
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:096758ac6552d7b3e1af49dd8bfb81662cabab1a6548873fe92cb36f70a3d4f8
+size 4425584
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..24175def10c59d7362de53e418278fdfa066839c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb7f6cb41c7223c507cb3cacd53e0196b4d4f02530e3ea88f1a87a2d3f101b9a
+size 3379827
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..44750d03dfd50e6654e9d48facb66e89ddb40714
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4f68c961308c598bb4cb673bad302b955e286cf1b92b7ecec9decea40d996fc
+size 4096859
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ea479b6b2c0aceae5ddfb1e0e8973b158c3ba9a1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2209f4d31cf0fcc31cdd0415745065333690d9eb9514fd63c48322ad9fa66d92
+size 14560617
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..36bb2ad8676cfb28343bbd9cabb1874e2d841fd7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:950194e37e5e78ef40371b54527ca7baefa0dfb81ec0b1568d2fc590f3886460
+size 11222846
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f2e6888a887e3c70c0ab904857fe466d47da36a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abbb17b3c540936fe0fa7c0b710c851a758830788d5e0c1dcc0da20076d62546
+size 6339610
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..681c05cd03261a7f91f9de9883491df2d19b4c66
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a449235f3f327efdc1b091c90d7104cd92562ac88507dce588c485e145dd32a
+size 4079407
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ade98f5f517efa83da27e639a5c1b7e8b1d26d60
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b223e9994ef6d3a8df15a6d9e8278144f537175a5251b646c45fe63ed6c2f127
+size 6367724
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8ce9c2e709ea25af7a3f222a3e32a9c15ee5c71f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f229412f1877be88058d10592a721a4bcffffa5d4d77855bc7d09e9d6197584
+size 3842907
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3124424489d627b5599a1d44eb120961c8d2806d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a96245f47c957f92bc630bbf72716b1735ec88924e3abe29be230214b6fbc287
+size 13662978
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c12b7e127dc98c9f90c2e64bdb210a6905589c47
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:548db8db86631373a1c7743720d8e553e4d901c01872ce1ff06f4ac7a73858fa
+size 10455124
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e75f1d369d40169cd815f64c736d6fbc6619ba40
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4399939b94c040a5ad23712f0989ec8058b405e1e7c0739468a217103034960a
+size 5917885
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9a10e6bea0b929bcbef3237c1bc183cf6692a4d1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00d5dabbe712f684aafe42999081c950b396ec0c5dfa43e2e260d84cebedd440
+size 4042506
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0aff326fa235cfb6521c0c44587723bc7b782926
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d2ee16b170c2e6293ae6dfef10aa6fe010dc2fbfc9ab1b2bf607cd1d6abf87
+size 7598986
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..313371ebe6eb21729eea63ab399e7e6e7d2a8aff
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6f92b51580dee93664d7d4f901cad3eb75249451286f1946d20f18e2f7d1de2
+size 4639746
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..85f0b1e55d8cfd4d6a6e61502f0e086636342308
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e4a1ab3b00656d2bc50b8b5cedbc1f3b9146a3495bc3fa11823962e763f11d4
+size 11030100
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f791b0c206df482d53c8a14d12173bfb06eae10c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d21ef90bd6de42843ee0418cf8a3171c2664fedfee9ab1eac8bf60e20468a133
+size 6392192
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5924bdf733da190c2d53e392b14f2fa8a91c845a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:994493fbe6d8bb60bfd2a8dd974ad1cef1ddcee2c35ed926b28464ea631b91f5
+size 7260079
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b1a9f5d8c9dcd665c6b775464a0ef0df3a1d5308
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a55d7f1210aca2af0515b69b16490e73ba8f4568821bc880ef6ef205a676d12c
+size 5139946
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..306042cc2e902d0492466b73c5269a9effadf45e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbfed85cb8ef672a9b1de92adf85174cbe2b3836dbfb4c858453374c4e3431f4
+size 10432550
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e35ae9dcea3442fa96f339c868d9e7d26b599961
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37042256e5c152ff1912f0e147eab3d5d1ffaba1fc9367f442bf9e25d1024e40
+size 6529211
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..79dafcf6c30592e81ee08fa5101238bd2cdc06a1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef5ce8a1e330f84444b325fb5881a12680a93b2acca68e7e6786ed41b9611969
+size 31483784
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0bbea6d30db59b2ccce1d17618470560f262057f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6576ee594bb083c1534f90caf6e9d61ae67c9a8bbcf483ee8b5328c7abbeed6
+size 18419302
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8e74553e0c6d126d34c8aaf5e82f726d6ae56260
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c674c59bc3b701cdb532fad1977521c70cd21038af303d281b8e0507879bc03
+size 10579500
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0616780ef083f3fc9632f483a8ded06d0a25cb50
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fb0ca3820334442b12d4c080577d097a19986cadbad90b29653b68d976e0114
+size 7949819
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3f0f3f73827cad0eef60998fbd2e02d87baaa2ce
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:124af96ac4af066fcb89b5f1c8b26d96a846ca10c05e2bf837c445b938b61d59
+size 27104142
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fb3fecb72abe68c10d55c18311dfc2e53a54762b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39c0c67a245af0ed048485c5c78871089f35b23415691bb5d11d5fabf5ae5e40
+size 19202472
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a5e935acf29628da86f4d44daefe73abdec9853f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9976b2a13cc4d3258371a9bd0106474f4bfc8da8dd4088b389656b87d3892ba7
+size 98724296
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5e584c1acd4fc3e57130ac62c6bdd45873d7d1ec
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f90a28c24b785bbb8839f237d1fbf8e3b76fcf3a9e3edc1c21477a907b8594a0
+size 29889313
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..51650b6d969c5ff83b8ccabe985eed24c72dd7dd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fadcd3e9037446212ff7066f55590876e1d17e6e926fe4f6dc1e8bd66b321525
+size 35291669
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7ecd7f8a3515c22dfba01d953d0f0ad5db0e016c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9011b4b7ae63ac41526b51972219b5d1ed05a68037f8b7507535e1d7882cee
+size 7739505
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4aeb6b32d8b5ccb4f3a6747d59df88f3740d867c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12f047bbb5466da809295a75052088dca8fc737eccbd0c3f8089ebb3f493d4c7
+size 26758014
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d5dbe6c947a9b45d0ad930fa751fa5a952d988f8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2011d1934231ed5d446c0ebc46e94c957944ebc1819bfec8d404655acdf3be5c
+size 19098629
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dc808c0e0b83ab9d6f3d6e6404398550a00f4082
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d62f09266ed4f70f6c305617b405256c7adceb14676b33ab896c1363145df399
+size 98571124
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f542342e73656362a5e8ca2a860644c6ec123adf
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8b694eb0be7daf99349c109ca7b98e076e01f3322731903aaf7b496c851dae
+size 59937406
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e2fb20077bffc4309ed7acf4e2674142e69d480c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1027ffda631562a5f7c4b29db7b2979432d9d877304d18dee1e58a556860f0d4
+size 35431132
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f11ebe44728293c48d4c44d996d059afd667a024
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcb2b6954ccf6d5e3b05b85a4b599ff53d3eb47e430958d750b44813487eb525
+size 7682224
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7297f42526943d2db30d0da421d287b496741f72
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be5dba577524f8bfeb4b1eeaa55c66f0b573611682d57a387d98024ba92d6382
+size 26832722
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..957d039e20b7eba162593d0c008cb3099d730719
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d036d46e01a515e9209cb61a438590381a5339a901ac0f2e572a73ef937acae
+size 19145250
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..06ae8d05d7abe2fb26990ef24b7a7b1b3a129014
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bf43249b842881e137ba41f9c7176d3b816f22afc65005343252a74969cae8d
+size 98787368
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7703490e02f470ad0bb0c2e9d8b2cb5bb26a003d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a49a82e3dd70d1b8c79150bc88c47a6d6e744f678b311b0c91a9cc4c7cb798a0
+size 60134482
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..58a7198c0014d3ecb8153916743c38aeef63531d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22fee52f506d2d423b9025cb7e277de1f3706b8691d46ffb0bfacc3dad216c14
+size 35572279
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e62ce2490f668fb9f2728e8ce725564ec48e7518
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c612f0b0734ee722af7f76c218695ee7078c3e0efa62a3e0a847c6094cef61
+size 7527981
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..29e488415aa77bc759c9f4f42a7313ebf98c673b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d56c962ce9a25624ca4ee8ce8e177d1d2cef33b3c904c73bf83dd8943296a5ba
+size 26152498
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..614e98f53f852ad462e2739f1290f30fa95d326b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aaefcb49eee01a8c095986f16be10293ee62ea1ea938b664cced949bb481971
+size 18735845
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..66368ec434bffcef033dc65a420fb12866c1f6a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dff49ac99d6f143160a852d237c055cd3ec7ab6b0c97de52a96d0a20cfce630e
+size 96819136
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e6092c0938f33760b6d97d78f070ca7ad82ec507
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:affd459e0546826e33c17ba4c3062213e3b949e19f4391ee5e34dd3fc4d394b1
+size 58862378
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..07dea3bfd919a7383aed021af588a58411a237cb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd43d30e5bdde97b0f69d82be771b7396c94c1f303d340dad5bbafb51f888d49
+size 34791278
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7a3fe7028cd2b3356b0e08f856c5f244ff26919d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:716f40abcdfd5831edb6016af3b6a0ccc061c4cefbda8bc2312329afeaed85f0
+size 8124741
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..185ba21b46f2012b1355b42eab66fa6a8d78b53e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08f513e1ca1d36fc9e28c4d1b5a82397b895562519dd16f3df360824bfb73fdd
+size 27681906
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f825949c4a2f1136ba7104bf6f84a6b9c6edc817
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56a2205f57b1fb535e169198f1fb0defebdbc29e95aa0df30bba00c9a75ae5d8
+size 19704639
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d99cae25df2459937569e9d5c845cdc70874ca33
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839ca566eaeeea88583be932a18854debe9f72b9e8184680027ad317dcef7d7d
+size 101367520
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c69a84e6292db643a05beebd0f553c69301b9a5e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0652d589826f7814487c182a1e4c35291f4a9e15bb49b7b5a842e9c3316bc2d1
+size 61652188
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..200053b3f3a4b7b5beca3428940193242ac6dd78
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e855ef7bd04707425ffb82cf346c459aae6ddd748f91dc530adc1ef7eec7d5
+size 36468074
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f902fbcf412b6202db902f6b58ea80b79d3a471e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e980e105839c7039ed4bd1ca4dd283496d8513fd7df8b20ffbf9a15fcda91c55
+size 994044
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7158442526df2c15db0545c5f15f388964f20764
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89b438b111b68da40c9e97cb6fd4c887773d18b0abecb2abc6dc7167cd648586
+size 1452619
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ad40200ff0aed5ced6bbe4a6b2e61c4ae102bdef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd3164de351b0c82e2121f2c01dbd73bdf0647e68e9a71f37b76a671e79afbe
+size 1910581
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d2641546b2dfe1bf3f16c3ba3b1cfa7ca313049d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2b6797c4aa5a0b45c1259179e8dd0505de5eadd97bb3d6b6b48a1f0ae6f1ef6
+size 2367654
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..302f08de05009c40ba202b3e6406bbadfaa7960d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34239c79ccabb9e8ba6223229a5735d1ee8b7b965874cd6a716d937b68ca17ce
+size 5643884
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..00f29d20cb60f4b978412790ada28ce98b8fc1b4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_GPT-3-style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:235119cbf599908fb6262e0b1f54cab7e5209f8458420a567a59652a6b063568
+size 3278165
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..073df53123f5cbd5fa37b8120e73e7f64cbcfb96
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26b48f961a4e09081ec1d1f91273d0551b15fb6ca5a93a8c4fbdbbf902584e6e
+size 1203155
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b1ce878e0baca92bdffc5e1e95712a14e4725e7e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:701bf2d17666b683084b60e3615d0beb15979b4cea0af420cfaf229c978efdd9
+size 1755006
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..99062f35dfc681a915eb13fc167a04e519a0b364
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa90f690d7d6ca577c076ae5cb8fdd321c00925a33852155d6296f58ae24119b
+size 2304828
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5f8cb8e8795dd8946d8b1598df17448c3aea62c3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a35b807756a4ea3646979af4da5daa702a82369e000f29963f12be0139c424b
+size 2853315
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..55fda159e26cbf22af1b6324ffa0e0996e0ca86b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657330573954b9f6d2022cfac3c28375c76336476b05de08114a18dbf8bc111f
+size 3398821
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f30243ba6a1a9682ebfda63f8928f3f4a9ab2db5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820faab3335e13f8e8ec554fd86cacb0fcb6ab68fd1a9be2c89942f731df8e37
+size 3946293
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6637612303fbfd4d698c922238a258ebafc26ad7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:735439682ea5468205419decf13352be4246277fd2452c4afdb21019747ba8d7
+size 1007540
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d17af39fbf38127572e6116eb1aa07fa5dd25923
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c9b4b3bda14064f7cc0e6f59e04865fc4917231c69778be9e3f447854fa3761
+size 1478580
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..53e60a1a73d4c3b307f17ab12620b335ff6a0b5b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95e58ca7a57911aedad39ad1f30b044d3e7647e4e4dac1dcb2cb337e49c5f8cb
+size 1949109
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..34a807bbec57bffd921c7c56909ece9ce226a575
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16df5cb517b20026b57022603f0e41902c1326f0ab58cd124fde5b301b24c50b
+size 2418545
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..26c00dd4e349552e2352026acbd2dcc4642c865a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c45c0f516570859358d7e981577b1720429e8a0df085287706e5b8a23a4c328a
+size 5769946
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b183942d5825dc9f5ce1f09194f88ee2f203d4ac
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_can-we-infer_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8a697499284706643d95a1daf64682e569677f27899d9ce71e5b5f179b63a95
+size 3353356
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fb31652781911a7a79c7e13be94e0ff7cb5839e8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7185885332fc572cff19580beb8118554b9b863dec3d9bb5b6f557488ca5934f
+size 1161372
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2a2d0fea04ff6163e789fc4cce1bc0d32b2d56a4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83bb7d884f85e5fcaba3c24dfcbfa55fd61427b95fa7dff5330b3e09fb7a850f
+size 1668647
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..81260b47aefff57022c6fcaa38d11bd9b24a7e80
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc25e9dbd237cc2c2f910ebf4e8164521752ea1c0cd632969802649d4764d5d
+size 2176703
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9600493f41e09ddc87f4378e3e1c5b90ebfc9179
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b4883eb8d962ed8b30ca9176cde8f2dab954f17825a3207ec7f59e9e386d322
+size 2683477
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..354fa3dcede3761d91aa2f079cccb99e84e977f4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e7d3db459ceae291e84d5f96eb08e02cb86f0d875458514a79d3d3115aeb27b
+size 6374716
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..712dcf41dea513652cf17f5f6f267392516b07cc
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17d804417445092c34fffa9a49db2eb42570c6b679476aac221bb4df4fad266
+size 3693309
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5135a97926377b1e644deba09d77be6dd938f313
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4e407acaf8ee244a00185f11a8ffbdc6ac2eb1005411df3209e3e7fbba06aa4
+size 1026578
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..282129931f24fab7fbad88f3711acea03fe933ef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f062f89955e80c59fbf2fdda933ec3669eca57c7ce2c7f9c2b29526c3a4b642d
+size 1503568
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ccd28b1f0daea33968b910e551687931fba72bb6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8b6f84b50652810d11300f59abc8225cb681bd1320b47251c6814337ce85b88
+size 1980121
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..058be0f919daf0fa7a15669b69d3196398d3338a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f24784e3c1b75b461b30923bf050fa9b72018d6696b5d991c40166ab0d67355
+size 2455529
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d7fb274c9721a40b151a0907ffbf7ca6399c02e1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af5af7ec6fdbb4575ad7d82937ab09fda254436f852599de540e202d9655c582
+size 5855884
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9151d194f6af9e960df71a280e7d8ff770394d37
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r1_justified-in-saying_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69f72683aa6098e0c9972e151b26d7da3a8fefbed0895f97c93b6218da10a8bf
+size 3402308
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..85ceb4154edbb033d0bc6147f3fe769fdc95717a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cca6c4a00923a17e320ce4dbf00e9af8111aeb5d3de0d3d1b32ab6c75913578
+size 994502
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..65d42f0df4b78978bcd4a8b4006b93d5f5874c2b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:842ffffa2218c8f1a740a42cca8c64b774c34c9c41d88072b731bdd1e09969d3
+size 1447761
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9337dd80464ce38a6594617d4138b31d6e196c3d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e4f442332098d3e38d01415821c62ad6cdf32d33f9275cd8e967f713aad6bf1
+size 1900563
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b03a4a1b234ad906f8733c89c351db9118cae7df
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b796da7f0043b00815743d81690fa6a82458a4a9e57a80553619ebb7462a2a7
+size 2350137
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bc19bd40e03ad135ade95344191da78041e25619
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed73e8ab7fa3251c290f163ad46305c8d8c7545c8ffa1417f80765dd4cbd28f6
+size 2799731
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..835ce883ada7a6dfde8c3048de7be1c075a5961e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_GPT-3-style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:139997e3ede94a2ba487edb5985bc019a7af99fabc222914b91bdfb2d655ce57
+size 3250334
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ad4799c2dc3c6fac204c9761a550cfe55e8844e7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970d2628d775a50c1200c1dd6161e8932777fa095ecbdc9e8fe6a7cf54fee59c
+size 1203533
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..79b9effc40be6f7a0ec61081d035d801ea9e90ac
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6600d531936e7ef70031daa10256ab0157ee2e4db6c4008c3be3cb0877b09e23
+size 1750064
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..45667e740d4fd201c66f3ab2ebdaa069b8de208d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c69603163d28dcf7ae28f2aac0b60072c65e357a4f22d01180771549100284
+size 2294790
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..23b4942c2edc6dd00e39c381b279df02bbe8bc43
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63d91ef680ad51704a2b49901a9db35406bbea6642bde1dacec71d8b5abfbc8f
+size 2835984
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c3bfcafe7a794c0f9d94b3cb253c65a6538fdbab
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a0fa637693799e1ce9f945ee50512cc2e5c4ca6c0d2facc47c75c35715d000
+size 3376929
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..90ba54f880dc8fa553d06518a6ef19573c6aa8a5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d087dbe829106b566baa409d0833e0423fa999c4c634fa989b513f4989d14b7a
+size 7837478
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ee1903157cca524a38250d7955f0c890918b7311
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b580ec8567056795c3f6442fdc6b2432659d0ba712dc4a24672270b7c287cbf
+size 1007938
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..18383a9cbcc74f02a35b9ec207cbf37c0b56eb2c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccdd9c1ae99fe5333d249d644cf6a2a3a3618925da30883e8e519ee77fe0eec3
+size 1474027
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..66292518221d7b37be0080b99d8785944cb7b89f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8ab3bf8be4b5cdf9254fd0529070fb666da017abfe6844cbf08c6fb7cbee145
+size 1939318
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..150d491ea801436b90d8964f77b7d901e0bb08d7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15719a32c06db35b8d9451ae7c59be7f9efa22ff0ad7620457d3876005676617
+size 2401286
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..99265a9cd94b3ad3a76bf407b28f009cfa79238a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f746dba4b8734a778ea0dfe0e3a69b27de44e46ee44cb7b285ffa373628f955d
+size 2862971
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b8198a4bff5b8cb2632db0e58b6c2cf00b12c887
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_can-we-infer_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b0b13c8053c8d53a3841adcdae4484cc4e7ec0a9b8711a77e6b555de67399b9
+size 6651388
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0aca0cabbe6c6c4db849cf6ff8015c56d697513d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd592d7ad3dc30fc84bd02bb4be5ba05e10bb533f51e065bb8375ed51f69adf
+size 1161712
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..02cc4d8e6a3692f51dae79a6e440318d6ddd35a6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3737da7224265c2db5b7bfa5c03a1719c6f94ad2d45312502d70995200e9386f
+size 1663799
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fa0b3f000455cae78ce902ace788875b66b88e9d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac30a204e14287954a31db9bfebcea8d2889783735ab493017d3e945dddc11f
+size 2166268
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ff0081592ada6e53ba5c02b27932ec331f1ca3ea
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1b58b7955b66a73567f68e1d81c708de52a26110aad6c1f9583f85612e730e8
+size 2665399
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..59a6616091c9c95593c658322f8fdac2101eeb83
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b9c8186319f70a6014f99becdb69d1e0dd8534ec326216170d6e2735141fb62
+size 3164391
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2417830c44311939ed9215a092eb0bb54fc77c90
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a009994ab38433c0529fa27ce9935dd80b7f0c9118828f99e9090b7aab6bbf42
+size 7329108
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..58e2b8ae28ea26df54f6decb57bb2df94aa333e2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93b22e16d66b4502306a1831894269387a29a6641c19d77dacd4a7bda72d1ed1
+size 1026999
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cfaf0ffd65f78ba703eb228b0495fde817c37177
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd7cbcac38617594fdd6f26faf4e63dc55d9068bb44e6011c798ea64edd7ce84
+size 1499000
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..57f09c9879cf3d1d0b5578e8f69d95164c58c929
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6eb057892fbb4396a4f36326643851c507ae77fdeca1e78b87905eae4040b89
+size 1970313
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5f2b552b9d417260f8dc9fb1a9e8f5ced9990386
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8660ee2598c2f50173bcca5d74c3a8d849ecc8897db0073f448d233f4d457854
+size 2438223
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ee173ee2177963ff3e63bee02dede37882af66d5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce676518bcf3e7b0c1c5df904ec3f6153811daa4c8f41866d45f6ba759b02b8d
+size 2905922
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..45230a4b1e13e250806ad706500b138c8dbe53cd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r2_justified-in-saying_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cae306ddf8d35fa16910bde08c6c45b144d10aea9dccd16f8fdff13e176e50a5
+size 6749292
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f167caab3e94aa5eeb6580fd7cf3a28e35afe58b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2040606c6df5ed63f22c4b52e8e11a22575a844e1dced7f56932da825fe8fbf
+size 1170874
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e67d4eff8e84f98e2bb20a86b3c17b5956ff94ba
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b5571f49c81f743733b9c9144ee834e634aa031b04798c44cca08ef584b6d3e
+size 1699228
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ec8b8fd7d9d41c63eec1356c04b5c4b7a267c7ae
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6e91e2c033edbcb7a887a6023353c477358dc65fb9143ced4f5ebd1f8b9fb7e
+size 2217423
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a444f46da2e14588b58f52b3eee188e803af4af4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab792e0294a15b2a337122da70ceb456293cb374dbf0c9ff86c62f259be02407
+size 2730563
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d9aa1d6717936e355273604e4043eb58ee398292
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c2888ccb85a8324f5f415cb3184dbbcf0aacf39a9dd3dd9d7ce8c14ea87ff4
+size 3247974
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dc7e78d5bd5aea7de7fec92b081324df260ec012
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_GPT-3-style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c1c897b7f715f8e8e53e3702f3faf575bdb25994cc2a332ae184afde06bce2a
+size 7553018
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d9ab6d1d19217ac8130db7bac0858eafcaa9b876
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01502f3b2038932229169daea40525f6468980bc4a7705c7636411b7d101a532
+size 1421169
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7b0887bf9a77b263cc09d763dea49d1dc626d9bd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75bd05b4ca4ea144f8d8d5704961c69da845814311537551c6712f7847940db3
+size 4123718
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b258ceb58a7e193076e14a76edce34b05928b0f8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8702fc67113eaa2f935e5276547be30ade60a28c672d8af7d55121f4c43b84
+size 2690146
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ad6a1e7c80bdc41d4cdbdc442257fd6377dc37ac
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ed2393f12a73b26108e4c0edc106c113f46b122adb8dfeff58bcd495a0f3839
+size 3312899
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2437699a2fa82977df88982399031184866b7445
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58afe8ab7aa9c4a6f0d4c376e74bb95f2a5307c824e68669685e15c7a83fe55b
+size 7879416
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e2ca7c0329dad6ce4e318427e35902ae90fea01e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8295d836c5a266d02ba1ec31e839e93310cfe183872a23034b6876d18328eef9
+size 4577514
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ae0b7272ca9af54d624875809039406f2c316448
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537743d609e87105b81f828701d831be936fec4fb5302f01c41695564f910ce3
+size 1186569
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..01eaeeaf1717323999a6a247a9dfefd392128bde
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94988e12c5f37571bf6f92b20b1662de84c07dddc4cacdfeb002f20698f4985e
+size 1730559
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7dd837af615e4a8b26a67ddb27f6ef0b0022d0ec
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e35291284b0a1aa70cb079f1fcea3145f1de738b2e0d38bea5e85823d341e79
+size 2263902
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ffbb4470c2a03825f6f5f78668adc023868539f5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7250ecafc0a28c018d727bf805dda6e9c61bd4885b642b5a92b0737ceb608367
+size 2791639
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5b62126893776a7db94b20fa8923395c303b595e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:917e1a0dd9f5f486d4e80393c3119dcca49374c5f9b6b0b90ad4029b55452588
+size 3323612
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..38a0a7662be34fabe05759f468ae067b67f0b07b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_can-we-infer_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98f0b0c0781e6d0dc1b6ca432373f92047e383a6ad5d27afa377135d3737b574
+size 7733034
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c384bd0ee807c10fc51178f4d54961cc25376dfa
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed9ffe6ef89cebbe7373733bf638c399a0bebae70df4a46b064211d86102202e
+size 1370271
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9d6350c3c037eadbad97ea6a2698f83d3e3e9f20
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fe709edafc14e80bc973713053c9e14337943a2d4e6748a077f646830a2a33c
+size 1959145
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..07d7794a69d0637010068d3d7f26a93800149ab9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbc00aecea8cde1e0628942b4c7b0f76906e4dcb3ee2ea0603551c6fccd58781
+size 2537376
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fe97be5cf8ae42331336776fff9379d90fc57426
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80ea63d046a3a4f57e63b7c29ac817d8d42c076f5e1c3dc901878d3003f3f6a3
+size 3110247
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..effef41196f6e1b5d8e04024fb2e527027c517eb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ec7681e3bf575172b44ec0d92ed350d9069b242be6bdc7d094e12c5b8081e8a
+size 7374776
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d13af914aeb60f867d86ac22d66c52aca7005ee0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1064c1d797414f10cb3e6f599201c6137984dda52954ba255d53279cfdc3fba0
+size 4275537
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9fbd9a27a51072c37601fe20ba97ef7b560c13bb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:141e2d638135490eb8810259666ef59e4e3dd9405541f2e489ec390da30b49f0
+size 1209422
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7766b9895de15a3cf09d1185d2d527deb423e0a7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd6e61ae863afadd71fb5d148ce2548851d25a9f481a3c54d6ac36f0da5dee0
+size 1760504
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..62e1f362dca9155009a1d4c193fbbd34db37dadd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61fad0c6998964fab407f5ca6d278224fe005e8f87e71407caccae356d4abb05
+size 2301091
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a6e72f1012070d8abdc040195c3bd872076fccb1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b4c49e66bd7280555c02a78d3f53a0d160cb744413c854e6ef8f540f2cbbd5e
+size 2836010
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..56a13071bd58d897afc17c38ff27af4509d0b757
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25de7e5fc7f276bd701ed4c3eb5e6200b3e9be8e6cca612d019d0163ac5b5cc0
+size 3375161
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ae7320dbc8320639e8d4e032785a7826fe9b18f0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_anli_r3_justified-in-saying_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b09b7e23997e7549ee47afb8d1456eb6799736c3070b67c09fd6a1234ca44ae
+size 7850540
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d56269b53c67cc213f7e69acc60192b819fcf538
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969513b36205603156812e6f2682e2a120e4bb1568b7accdd30da73e228f698c
+size 1216777
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..29ce8940d7b3736d5fc60d022e896507b640733f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f01deca4e226b11bd7e4a8b0b7fce4760ea1c97d77e605b692a48b9c7bb83b17
+size 1670632
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..77374f0ff06de69a65c03bcac98093ea527db9f9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48bd681d388f1bfd969a1ca218311b8e50106eef0498c96db176947640424ac8
+size 2120084
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0936b1355e204da6a95a4cf2bd4664498854755a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92ff7721943b11c0a78794b483f3f8de9e309c809abfc667a15fe36ca21a08f7
+size 5155230
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a0e02167cd0859ac803757196c37a115799a5947
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed178bb9a4dbeb3d108272dd72ce06d79258514af72509a2ccc8f4180121a37e
+size 6055034
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4048aeeb9417470394df1d8824a70db7f0c05269
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8f840092149af5e265a201adc49d86355c5458f76ea4314d6f3e0303d233e5d
+size 3479076
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2a3e02e45c6f0afbfbec9be4bd729edd78331c9c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc1607674d2bec575d8c4bc5e578a73c1828830e3607281da2bd0ad53960db1
+size 1458162
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6aefe4056af0cf1dbc40d930c67c920aef84fc84
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac0ce0ee6f196d3c50e705512b0ec18f6d4a40114dbc5dc7038fad846047ac1
+size 1960437
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..28d2fd034f2f9b8daea9c7913312778ca851b195
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:714d9624c9477f34dbcc5ab4370913fe515843032e8b84955fbe52743093b38f
+size 2456942
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fc92eb6b5692adb320ec86cb7ddb52e2dd56b690
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba115b7c48a10e55cf31b008a324760d5b291e20229fde731e6b144161090f48
+size 2962723
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5521a53aca36f440c45455e4d9d3660bfce3175b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f041cdfa4d1d01b8bf0172a9707d9a5bc5ee4c5acdd50989cbb88a6bf3496f14
+size 6920890
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6980a5241942198882c956edee561dffa997000c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca370243ffab44b45b0c9072b6828ce42182076739f4af9d653d980d217b53a7
+size 3959551
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0965875d9543f859d9483998028326f1e62a6a29
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:086e7073ae7f7cfcfd45075e6672e0eacb2ae660b5a027f8f823abcb87bc2868
+size 1505834
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..321ae8644608bdbbde398499e0a24ea55b3d30fb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e806e5bb2e08ae9f4a67caae6da851f5ff8a72fa3f9c17d758099ae863858ef
+size 2033272
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8138c0503b5c447c8ead2e16390134df389c3970
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acd7684d3ddaed3d4200384e34d7b0c6078762d9719a599743434bbfda532c46
+size 2554333
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1ff4efc8c647c3de8d90f6fd7a0c297585ea539f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28240c62d79e44b6acb0b48a15ed15989c3210821c8a89843cee9d7e019cae58
+size 3084471
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b96c9a068b7ce59a324636ec1cf58f16ac904c20
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ae0ea095f36f74980086b41d5a574af93928664946545a6d1f05fabc6d6534d
+size 7213884
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..49bb8341b8108fac15c4e56ed63b127cd10fe422
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_multiple_choice_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1e020f6bc719cde4b451193215cc5e7953f7326daf0fe32c1527dc2569d88fa
+size 4130578
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..15c09c4cd505ca6c9362b144922423c396a1c4a8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:454bc5d9089f81efea46e464665c86c544fc6f4e859622f29283f4034c9a7554
+size 1202714
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5d99144556803a8cbef88730f3d6bc1933aa6c03
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5d82a84c1b1cc9caaca961a1b70f4f798ac969b8af522d7bad9ca45761ad22f
+size 1638992
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0c97ee7f5494565c6e7c1f2a86393f2ad347d447
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55a14391c77b4831d6c339d3147adf859f7a6f6ab7d02f697fcd0d8fbcee02e6
+size 2070864
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ef706996b5e90502dfde4e09ceb7e9358e328a86
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c65092bddc6636427e2a2d0fa97a8599eb91382c0df65fae4e315eb2333e155
+size 2510815
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..13bd29285dcd1ccd0e4209ac89b7796b2527c5fe
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5044045d9c9a01c04a3c2bc8c7beeea1662a05825429530d66cc4cfad39c3814
+size 2943137
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cb338a33cfc47adc64b3cd44275e446e26349e3a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:049d0ce77146af9808761eaa8637fd794b224aa6693835c7f06581738650c172
+size 3377116
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d2770732ae44161baf101390275105b67868a15f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:154e226d9f95d0697f30c34e3c0dae051b0d8f94854af5a59c5792c031042466
+size 1187348
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..332f1de709a678e58a2f727b5ece8930d7af466e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d57afb330b78412d9216875b9569f687e6b04cda81879e4cf4e886a3c0c7e4
+size 1557151
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..307b371271a4e7a05b8b433b3b2d6ec4cb3458d4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7f25e769b2370f760bc966e633afaf07ea2240118d525b5a7575617b0e258cc
+size 1921264
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7343beae08cf6d1262e564697b21cd2101e353ba
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f5b4b473de0ada18e42f53b8bcc4dd7bd368d4190dfcd624c86a675a524a19c
+size 2294455
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fd5dfcdf02fa4b27b4875e277d3179a1a13ee3ef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecda55aba6ee3b6d2322397d7642b361173c89e9780b06da62503b54ac8bbf70
+size 2659974
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3a0c4f46ecd661e90cbabce8d6732a6a9f23297e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_challenge_qa_options_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53edd2300bcc92ef4217bb98995d11c3d6c9ca2f0286eae78a87974d36450455
+size 3026687
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..aa5ac5fd370204514eff1fafbc79747955d77722
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0535f63aa628dcad7e5a91842b286d11512da0afd2d2f7ae9235345f714b542b
+size 2351241
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..23faacedd4760446abeeaa5db22592c68030fd19
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc2d48d5764959a68ed4cedefc9824975f58b74614828816a6086862b835e1d
+size 3174164
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b4a920c0c66847258e9479f26090617afb746e54
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a201d4a0ebda0c87af891ac055a16eec2873d13d7689d97fe20ab65aef8f54a
+size 4008946
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..68700fb6272867ac0457ff14fa0d42e447c416a7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f31b241d75b450168da8c9c94628a3c2df76eb50bb50540389b28be71ac99ce
+size 4831612
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3a0bb964490334a1e4b330ef2e400e627b69c714
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92f2e5da3e9a39b07ccf2e46add67033ef3796fec8fa1db1051875cfd95b6b2c
+size 11324712
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7c610053d9b894648699c73392e06153f5402862
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_heres_a_problem_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45629da40f88939d394fad3c9b3002b4ec9ed151bff82fb693c44ff54abbcaf7
+size 6494729
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..830a754bd15680eb19f0304bd392a6dc917a56fb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fdfd30ee10a5d6cf1796e91054b8f31bdbabcb1335db4c71241fe0feff1bc48
+size 2745894
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c92c00457370b8db7ba2a63ba2297a2fb7441b9e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a054d4567a29eeb30a75e3c9c98eead9494eb410446b95c988c6676b84fb962d
+size 3648519
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f8a6cb4495b840e8951435d445d9c7daaaa89c2a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d306c1b3ae8baac4213be0f8f064c30149cf5a1bb73ec1bc4b3d926e42d60ca9
+size 4565524
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d8d7ebf918bc5ec390f0913b339ceafb44e98e65
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bf48cdb1dacc403347a9ce6f07463169e344fc282eee31b0f563a89da444977
+size 10937134
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7e4363bdaf4c34c328dc80d6b5e1c3a620565546
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5999807fa947e29749e354d69363ce0bf1ebff3e18ac7d4962d8e71d1e24f522
+size 12761488
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2849781e8c48f12f22a92ff78bef184359d0aec1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf7d1b494045c99aaa78d0a56d04d907f4192539f0fdb148e6a3f7e56505f7ac
+size 7293000
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f3786578ada54aa10cd3efd0300fca3d693a9b71
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e956702d288959abd2cfce335cbc1b636b61dd48ff8c8a7909fa127059c66e43
+size 2843142
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9533a3f0152b87eec893a789f2e75bcac85b2137
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:096761760da5922254b4636fe684e59243506382aaf3c8d96c436ca32b154a32
+size 3796481
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7317503cf86bacdca1f2ed39f5052874750e10ef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:568b5a909f7b90c1300f45bde1756773a2705b0f7014ebfd11a4f05e51c76040
+size 4763535
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9d8c4533a5a8b5bf1a646d624e68b516d20bab26
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31790d4da7e8b2cc413f7b77b64d0fd96d80dbf2c7e7ee6090ed0d73fac52494
+size 11432208
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..916f85958b046985651640edb73f609b07a33a06
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f4a4de2f378fcbdf04d1ff4f1b58867c7b73edf932980be64e5de5e2c8a490f
+size 13356740
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..467cb798b8ebbc1558aa375807b9d408f64237e5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_multiple_choice_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31ce3ba997c5e35183fa834865674f092ec7bb21e51b2eec7dcd41f592677c19
+size 7640382
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2c78ddcba331428b6dd834d4e9868965cab8aa09
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d70fe0e9cdeb45b343fed60675ff52e479289efca80120ebe8540831ade2419
+size 2322732
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d8c89d873d11271721ccafadaccf044db1e4c64d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d2d7aebc048a905ef402d9d3b8c63a7177d29f34db7f9ff1941070a46247190
+size 3110021
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..404b027057d5c9eb431b46923f179d5b214ed231
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a389805b1625f9b4172bfd1b0079be05c34923411722bf92970472baf5231551
+size 3909160
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..06453e85d2f7998ab68c8a0ce364a95b480746b5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:093af7acd14e33c3818ee830aa766ab5b070ca935a710b0872d3392aae6a77f3
+size 9392368
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4ec1136791e0c49c94eb3b8de3ad004185f8b7fe
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e592c685ad46e6cdea35f70d56a56f5bf1d9c0e56e3cb3665b5e0bc53808ba07
+size 10982580
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..80b6ba25184e5d6b0b7d4608f1515115b1726f5f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cbff55d34e205ed1a6b0401fa7edd58858fb5cad8db27272462342ff460b6d0
+size 6288023
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4527d4c912b7bc142a191c7f55f7152c90770114
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd7615751d865d9512b3fe702d56f6ef235a51e040429aecf4bfe90b864faee
+size 2196841
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..22089980c4566a7988624d4b5db54ef88e1b9cec
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8c8328af29f2853f0652da3b32c40831102ad039c1d95e31a058f4cf8f3b32
+size 2830646
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..841ba278b08798bb1b21a998c24a24344e17cf48
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bce70775d013c2b9d0301f2f87a6de65ad6a66669d2d1b097d0bd660241d8efe
+size 3479358
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a8f9f34144341dc53c1ae811281e806756aea0f8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2201ac99991b229d7bb391c47f3b498f857fa7508d6be3d18aea727192e98b2
+size 8227798
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f848c74177a213496536e6a5aa342d404e9e2d08
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff5e70500c647eac4c3fd488c1b318d19fc53d6222cc14b66f00e81ef4645b86
+size 4757591
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..814f1fbedfff88a81d5097a61a8ae51a2a0aacf6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_arc_easy_qa_options_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cad868a04166cf9f3a094a65107c39319389bc6626d69fccc919cbf04864e3b
+size 5401437
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ff4af04c4a8f6fcf5dd9c7a32744a7b522b68ea5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8fc24ba8a384385474d98098cac964f103a1887e1f75597179bc558e4ef780
+size 3641177
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..54ab64952a8627b401a647c76381470184604230
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:137dfaa76633cb2dc144d3bb544d65cca56c7b78c88bf6cb6cda2ede6ce954b8
+size 5657487
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cf33a59d75d6f83e3b8abae032e266649b11b0c1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d07e9ac3b7d3f91df5c0daba56662eae69b1414cfb2b8f215d85ac17e00b5345
+size 7694486
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4c6047d2f0805f0399fd0988c7881fa6be1cd690
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8683dd2670fa27645021c493086b8cf87016a16c809a88b7b59b4fce0ac9c83
+size 9728693
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5d23579d3f5875d1b541906c7e05fd84f43c3cd8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d5cda2cc907955643cb2d8211042c4799bce10ea68b45df873b423424ebe461
+size 23538700
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0d61c4b880a426eca853df1fda4857e91f4952bf
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_GPT-3-Style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b7059d5d29159c9686ed59d278e0027bff35ef8a1929aa86ecf8d6dc031eb6e
+size 13790985
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..da8c5b590b0528d2a85723ce5c27e78034d64bb7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd929a63eb499481f1db6ebd617d1fabc32d73332e5264d123e575484ba0484c
+size 3984656
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..283de7482adec691ceacdeb0a78be68597c25fd6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c46d1b9d6651c76109a44270110decd02d43f142dd7ba907bbab36c011207f2
+size 6168126
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..33569a8acd9bcb8ff6a33c7a317db6846310e959
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35ca9ecd5c0a7449a2f7c160873decf04a1d0b87cd7c6d780b89d1f1351dfa23
+size 8371936
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f0d44258cd09033bfb94c3af9bbf935b568b9a4e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c782a8e598282962da07f7171407b2ede207e29877eba37f399f2da922baf2d7
+size 10573346
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7574e65c4b34b344191eb960fffc28b41b53a377
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d74997e157540703e3414358439a49121f425e01121e0818b3084e8d6beee4
+size 25562086
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f9ef8d65ea3488c93e7a67670afe34dd374986c4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_after_reading_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67d9bd19a5d590a735dcae2eed950c18e6cb09d10ec962e0f5eb739aa3df91d7
+size 14969796
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4c5cc738b45231f39574146e34348d5e888decd6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10e92fdd37b38c5e83033a73282465b497ffdd4ddeb21da20576031819a789b2
+size 4041660
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6ff03f8c1949bd47180534503169329f2fa86855
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2531df5a83637c893acb176bcbbdb0e97e557f25452f0ae846b1563645479ded
+size 6261062
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f9c7b28642ca0cafa4b034c1225cd243017ea9b4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7769d21cf4494b41bd95b2806611f0c9fb2ff736c6f20c60d41e45a5cc137352
+size 8501036
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b544675036df88cdb9dcea3d13c9c54af258b8eb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a034ef8b50bda5fc13a2be5a322530cf4551255b68c8b5cdbcc1187fe42b907
+size 10738340
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fe3fe0596992a06fda8e2a7f92fb1d192410288a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ddaed48e501abcd098e5486c064ed77f17a308dff08f2e02c32ebfe774a898c
+size 12982129
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d271ad309ad1ed37fe1697f148b420b4efbb2dfa
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_exercise_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2b5bafe36da7d46bf46e55eb14fe0b0a6f62c56787c416561625e8c62f5fba1
+size 15206907
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..873cd8904bdccf1442f458dfca51826d1a53dc4c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a51c451a54b67ff0b52072f7ebaa0e0c1cf40e15eed03c02cadd697b7879ba
+size 3663677
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..71b7b58337c70b737b7c4aaf14464a184b2de709
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:601535789a04c4f2e9ac4b10326d8929290e9359c1acf79c085e66b026f5a196
+size 5688138
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..35d0efdef27b0bf7b07cf5cebe692530f076c5c4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85f4fcd03f5fda08ea83d7209440048cb3f80d9b426f30e63d07c136b0953d98
+size 7733241
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5b1258cae0bc37702907be92004479d456819e3a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6c3d0124de615a8323365cd9f3f25cad66730f6673b8c84ad81a369354ee7ea
+size 9775519
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..705f1dc2d95518e6381ff31209e551e6e715bd71
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7594b1daba44e7bdc684bc6845493985c143cd9efce4266ba0923626a155d25
+size 23648476
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0cc0e06a2419b918c1167f6b25ec65bef309977d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_valid_binary_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8f066c021e31f3a7e23b6ac95584ee703d6ea2214a5b50eed28b738527fc6bc
+size 13854012
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d46e016b81c8a4a446370861ece4f5c04f0ab0da
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf4b355ec31222f4c9e05fd72ada57354caff6e17263de22cc70ff5e210b57c7
+size 3862390
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ca46151087c295caf9d09c1a5d2229d7e35811e8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28080cb1fb521b35a0a774f4b4d1498023ae324039056e2618fd45497a112e51
+size 5990549
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f527b849e96d1558f30ce60ef81cf6acf8eec9b4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caebf28056f0ceef2477e36f1c2a759b9c29fff651be75c4842cf210afdce1f1
+size 8142145
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9c2e2e5fa4cf0121f8a693dc55bd0a908414a35e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f64a81164a230e5591340e35141860c6410d76a16ca8d4512da88591ee02bb33
+size 20580730
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1f494ebbf4d6a3b229b61a754092d5f1318c1ebf
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66182810a4b2616071f25f1006cb3687c83c667f8d1f071dd72dfc5c5430d8f8
+size 24890144
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9c92b1bda4e424b2d424fc31d97920206c97e6d4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_boolq_yes_no_question_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06582c6c47fb1256c8ae794323bac2ae57fb28e0b6f4756089f91df935d18eb1
+size 14580641
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1d73bbe9bfe5f67a37b1d79498e0f8254acb5cb1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51aa3cbf2664243e7ad43e79c025ab7873f479b348e01d4e23d3951de5a54a53
+size 55217
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ff2c512ba4fe1329e6214f819d00fe2ca975ff7f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09bd338e10b22f8681cc043cb78d6c90556a6bb03df0535c49d914ed84d1494e
+size 77968
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d8aa9924fca01fc45f5e36a01ebf9983ce07fe09
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c8cb615bea084ef50f9659304330c702d120bb1fef6c14d671bafc913b0479
+size 99566
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c31868a6bd8a13c14a6d3005dc7082b15475c914
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62068a2cd0f5c98dae8834fc5fbc9330d5d349a673e895cfaadfe13b9e22c7e5
+size 120723
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a615cd1ba3e00ca57e287bc92ceff28cca529d22
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a540b89c0653a5699dfb96ca9637c8b75725427f094133e61583a8c0282a613d
+size 142843
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..72faa0d1b2638fc4224866c3084690004e3d62e2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_GPT-3-style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a006ef148d065e2618a398b4f11c84669a2a02a7f4f5a131fd9fa92b1fd0b990
+size 163669
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..806b11a9c8b43716ddc929108e3243bee2b45fa9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09d06d81d146b2e5697d769fd76799731df2c2d70081347d8168f300891d6b8
+size 66218
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..05e727f552de7c8f5595e084f9c4ab9b04849c03
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc9c0226b8c01919ce546dc31266c1d747a99ddf5a4fc3b08918c855832e043c
+size 94141
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..76e66ade781da7158593aa9f156ac9538aaa3bc9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef0020a5ec1fcf804bac10e93fd1be2cef52080c6f7751ae6b476e7500fd437
+size 120854
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..aca0baf1466e6bd7d29bd0d7fe5f8505c7476c80
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cb2e122a53701bee65b71b74b25ce0504611ce087c03947164411b142d7cf44
+size 147089
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..807e8d3a0c636e9840b069ff01386cf7196ba85b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf1254610aa99165f5f88e8492d6e5d296745a581c538e4334af509174f6dec
+size 174293
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..453d32452b432a28056499c54dad1a10acf525d8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_MNLI-crowdsource_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9b465103ccf10c146f523b62b46f414016b476aa24406ba4d5fb3b36ba5c37
+size 200207
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..00dbc56aa0ac07cd09b9eb16f140ac7e873c2e1e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5c080362c14ee60c7e0e1225d6f0d87dc62ab73fac5499443838dc3673859c4
+size 56284
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ff44f01ebfaf203b0aadbbb4c736951d9a69300b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:435c11683bcda5bd0cd70ddf95223cdf3cdc5b85f16c5184b731aedc20c254d7
+size 79778
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..046b4563cc90da3f08283400c49dee14b21bd79a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d9dbca0fef364537f454a6fecbd68c09e355e416b5a309d2337a719df34229
+size 102067
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..239e49a66959323b492c03cd34996622509b6d7f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dbd9c00f2182fc074f3b8e178136798f9f3b3b764e64c245062d16fce7eecbd
+size 123891
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d498ae2ad8811b53b6a86fb9bf0ac9a81c42d621
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:238ce8454b65c2cfe4d08d9481f6e2fae9a719af04ec0a7ec81eae0758f867a4
+size 146674
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0337995a9aa0cebc6159f9af5f79f3bf493fe03f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_can-we-infer_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965051555c36101b946c29ea269fecae5c1ddf541fe86befc53224d321723a10
+size 168159
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e93d37f2f368c2aa48246e061a331cba52e8db71
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8a89394fc80669d6184e7556dfe35e003b32db6a28af12fc4477a3198a96e7d
+size 63931
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..359574cbe38cafff662442ac82adc74eca6d9ab9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae71f03468e292dd477a7b0ae2cbc114794347c14c8b24b2bb874d06a111f761
+size 89654
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..32fe9961c79db6d9acda7ba6726b07a16c506f9d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169a1666daa324d61a0bf2b24197a08c41f515fa76eb9e618d632cec563e2d4c
+size 114127
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..36f6ba288fad09f9fc5e91a0bd80a091f934418a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d96dd8f6f0589aef01b894c360767351f88896261d44ca2e9dbc42d4fbe7d0
+size 138146
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..101ecdd453cfe0b7574cf52c0060979c0ac2f8a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec5dca76a8173afc77e6294fbe0619b4182c1aa28c1b4395832b0c5b0a3c1a4e
+size 163134
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..355d49f121bebf3de851b3517f89fe8eb4250c45
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce1b816911dc6c57017ad77dd37e6f4444b30cd58eae6f0d238fd44a310bdd32
+size 186816
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..68386f005174373c2c0ec8d11bd5db90491d9ffa
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87ee517e62ce1359a97b4ac1bba9a95ddf547efe951f3b74a0243545fdf9cb66
+size 57295
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..69c929e96b199a4baf086a6076e951135039a22c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21863bd9d7d9052f3db056506c886c3ff6dca9b9840cfb77381385cf7a5895e
+size 81124
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1a23acffcb08dc6436af246bcf0f48b16b2f7664
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35a2e2cc455ab4b0bd55098a4272a4ae9cf575def0a4475913b5730e1aaef3d5
+size 103747
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e3086c1044f815409ac1fb4e9cb304aaa87971e6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b97c243a774208972351878db4a0802465d964f19f380780e10e97db92eda6
+size 125907
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fcc56d91ab89cdf7b98c722b8034cfc19c4ac3b6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:289df201e14b832f1d1b945bf082d44a725dad6ab5ba44c21fb2911a486998ad
+size 149030
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..036d315512999f9ba08dc20a08cc67c6eb58854e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_cb_justified-in-saying_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0c392c16cace4f80a14dfb2db47210c71e40cc69dc19ab03dd5e9ce2bed909
+size 170850
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7c095347911a74a2f78eab825f707d0faf23e702
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:331e4da3897feac58fc24ce5583d4f728dfca5ea051fb54f63a0abd258beff6b
+size 92203
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e88374e2136b58d7fd2734490060017f59c48d11
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1a96415e5833876815ccbbc4e4b0f5bd16256c25e8c2b2baea77c52b3e0b991
+size 111601
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..84b447dc2cc2a202691a86dc1cbe5f9db2b995b1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876ad925971d212f561b1501f05f828a9d4ce74f09a94a7ed12ebcfa501b5348
+size 132109
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f911c4b812b004eaf40cacee172ebd41fa487a18
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24892c11bc74e121e6cc5e4def5f3c008ae4000ae316f4910da7144c9762df9
+size 152211
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..26e2a6407b06c8a471cf44fb0d75d5ac1c071ff2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e414d2b62ffaa10282757df5716c43445fd85d39b5570d2c3870d8820d7d442
+size 172051
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7fa2269cc53c111065c158f4c04330a3d2832886
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_best_option_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87bc9e6073663c0a4d9778139595c10dc466e2eb79ed8dafdb4da63b622af4a3
+size 191973
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..89ebeb094dd39c1ea5a281b4ec6f6539bf98583b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da350f28c8b9746d5f6cea06c7cf97265e7d12ca261da07f70488cd9e2bebd30
+size 87825
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..de85d625f8881d76e4a1aefce320cb19a142cdc4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f02b1faf89153f11dc719b7defdb475070c41c0b2d6adf25b106e629eb157f4
+size 105037
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4a0f169bc1e1f5fb1bbab3fa86377702ab29b43d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b326db79127329d004018c4461c351028f658e23d957fdf70d2494dd38ded67
+size 123386
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3df3106b466535d8084e6a9cfc546aff6a296b70
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78525b520ab15bb11727a4b83be1c8c46d596100dfc9705ac93e57e4141d0d5
+size 141375
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..16ea6fa14a4f4dcf11cd7e2e6b0c13c6f513e6ef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c5b0752fdf5124770698a83fed73740068631db6ad1dc8da6bf0d1f37e9d31
+size 159048
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..332ddcf5961ec081aa4f616c6d01be66441eb90a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_cause_effect_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58885abb024e2a064689f2877aa2b4b07401aa2058bdd9838380ee2b70839a17
+size 176848
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..28b2fc232801a4280366173d1b9c5aa95389b982
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b1d765fb11beeb3db89e92280e2e77e06d0aefc85efcf36a58bb2fd0858ac38
+size 85141
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7a1dfa3c4d7e13a62553163e54fb7ee2094cf189
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd0decb9101b5c3da180c9320754274073f29f87a63d92367c1357bcf3ded23
+size 101215
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5f3a869cf915ed05c206924f13e9dc602b9e81bb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42f2b6761943a107d68a54bb0970a30a129593ee373f8e15b2fabd27da0f89ab
+size 118426
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1ec0b099db348c039c3afa29c1bae48d2942ac4b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d0a11cfe5a44054babbc851cc3889257177d4304c623100487684a171e87210
+size 135336
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..debbefb56779fe02ffc552be64dbde601d3f349f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:657399c518af59c31565709db415bb25c0109f80b07ddc927f9b3101523b3b58
+size 151879
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f758e1c3bd3881795716be3f29735ae930ff3a36
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_choose_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65269f0ae40c7b0d3d36f1fcd23dad6ab3d8a3eec1247be86cdf8ba59f1b6ea3
+size 168686
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2b9c06aea133b662b8b0848c18184e92a5d8c235
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13358b79f734cce4a822165ec012cd4b33e7aec7445fae004c2e38ac3c58711
+size 96737
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f1239d750eb79e54363329acce3e2b56540010b2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98d1270ecd574b89d9db3f9a102aa964e5a9af96201108643ce7902ec228a0b7
+size 118219
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e6d6f6f0bf738aac1e3b0ec81cd5332d83fa7bcc
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5c8f7065599afaab620023030e7eb937bc1a27af64baeee79cf0318d646b4e4
+size 140874
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3faf6ca1eb1df28ac1a7aae0c4dce94328e0afb2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7a4c7a478ffe6e97fcc78150ba9a940e54eac0d1b2e8f419a9b567899955a21
+size 163168
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dc5b6d9c6dcd1d0de012ad34aac5dbb57fd46457
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4415684354634ff350fffe4757e010e1b4c664a9e369877e409c6b93d396bf42
+size 185147
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1df713a65b84d217928628888846cd7c611078d8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_i_am_hesitating_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6666a35b4b7d7f0ad868dd64769c89f899ecfb2aaad381579ccdbbd1a165ef
+size 207246
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..80fed080e812affa576d1d00b1285782f97200f3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee2ac5c07f5787defd481dbd93da618f67bd126864102aec96e9fc6c3c5a758a
+size 95754
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..26a610a284e49527c0054cad680879eaf56332b4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b5f92abafea57b003f1f7f0d36cc52173e000008457cd62aa9502e82a2fe9b
+size 115611
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a55bd99b595d30e02a49b79bdad97dbbb7394976
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b426ddd2f81647377d65116dfb99b55d50f0c913e4b61dcbb8a819e5d5e8ba70
+size 136634
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cf5733e2c1e19f02e818487df422816c353fdc7d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98d5059eb7624d3094637945c0e6dff1c0951928e77dc04ed3fc6f36a5fed74
+size 157339
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a2a10ab0d64d41498c76ab72da9161901b2d8cc2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b98b900d79dfb01edff58a7344fd65ac264d605ca9a363d74dad875b6cb8e131
+size 177704
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ed9bf1dd5493195262373e6a4371f5a375ac793d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_copa_plausible_alternatives_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe0219005e3910cef795495a20cea792e8822fc414297cc3c0b0c80d3ab2e57
+size 396582
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..58ed052c134309130626bec7e898e2c027adcc9e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f12643fb7dbbfd4c21053742478df6d2c371fb69cb5369aeb94ba49dc4c78e7f
+size 3446760
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..19d0a05e724321548b2d9fb72b92e7597a81cbd8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:475d85ab44083fb4101b9a3fe2a58af02969ec8495d30ee616c6c58e62627f4e
+size 7681908
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..592632fcda4d7428c1bba0b7c2585ec6d4d93670
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:613fb6fe7f21a86df9627cc64c4bee97f2747f1d533a661beb52b09ded67b751
+size 4885589
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..aa9519fd976d62946ff9bd030f04d0b59e3498ce
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bd53ddf60c8763167b0f290b04445beb80d4fb773e771000165acdbfd3fae6c
+size 11714696
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..925335246d131f498a44626251c943dc8c07c370
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5e60364f9b4bebf2ba3d1d5fefac8308c413a53334274b1d6aa727abf0df8ec
+size 13618086
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f7c4315830cb9325779f69dd2dcb05c6553cd5bd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e4cc431dcf4013dc19e8a1d45e6b7b993222926cdb338c11e78e197793083e0
+size 7761106
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d7151e79c32170062094b0620ddfa958fb559c4e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd0a22ccca585cef62fdd1bff8686eb1fdb84304200e08b773c06d9650115406
+size 3314122
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8c3358a7449ca9c66786fd9480c7638a0128b93c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2322d38cfdf39c560c98d996cce85251a1db96cfb4538173e53eb6692c54dda
+size 7611904
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..59802f2f5320763164aa1b594886d8969563a46d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ae5634fe5816497a3f8180c482eb8f70d25862350f36e556524c758ac66e68c
+size 4804616
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cdb6e74a8c1d4afa23230912912c328ca1a2c5fb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89412ee5b283da6130385ea17cf4d7373ccf28465f10addbcc306f76a98ff56e
+size 5739138
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7abee9291be40e3aef82328d4e1a7a476a3abf12
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ca3a031bb9624dab0487e17b779990c9421d36c6dca565c95c4064a2d58c251
+size 13315212
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dd5f7dbe0585a6b5c38bb7bf00599bef7fc1682f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3476a64e57362f3eb215c5b23522b6d761a9ff72cf07dcd95023b37a61b88aa1
+size 7576446
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4ae8e979149d92985b4d79197639d578d6cc63ac
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:179020eab0f77d2b982035ab3b539613491525fe3a35e495c5c77f4a6508d2be
+size 3791842
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5b12f0bf1a00f1c350609a33551ab6178ec3be32
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c884727cdf9f89a95d214d3f4289bb871febae18013e18f940f1be618fb4e5e
+size 10234032
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a46518ff0017e37237616d54de3246ccc7726e2a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c190749ad971af200b966e20e3045905b80a8aee20c2146f27c8b3f00de07125
+size 6152309
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..41ee47105610f7d9651451c3b363627cb42b1d52
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922fd72e9f1987842a0970a2f6158e4acd954b8b383924d88fd409165e09d0a2
+size 21534396
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1eb0a06f43ecf47e1269cedc5eff881e448853ca
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b16793d6445c57e098861021489e25ca59beb695d605d535cc176ecc87975eb
+size 16415392
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a8b13a4329ad44e717afcaf8b79dfd1a37d92a6e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:480221bf1d31760a600957da1d9bec3fd1d8f43e66fd264df0bc33cebe7296af
+size 9252146
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..91af844461e578d9110146749660997ed29863b5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34f7d2050b1faa2dfe9e55ab7aa62b329bcfc238022ca4cdff4d0cc173709c85
+size 4305020
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..10ad01c85f313ec91bfbd21aae81146d4caa06c7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef056f0741871b38c82ed4d9f5d2670abe17dbe4191879cfc7acfb6c5b99176
+size 10065230
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5a2ef333f6ade48844f8385e48acff6e32aec5a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb7596c815c3127d020cb7ca04b201678ab400457c069e7e56d6aa73fbd27f1
+size 6126397
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a543d432566b20e9b6b5a0152d37bb6866e4017e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d61f01cc3790fa3293c6104c772b0fe92395515a1d9e00ca83a57d6c41a3c05
+size 14413850
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d90ba940e75fa935b5fc499b23abb008c923a388
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0900f9908a2b0c8f371a1a0761522e1a31d6029ee3524fb68a9521ea6a2251f
+size 16567002
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..99063765c859f1bd05c419ede1b8ee171a629756
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0545c5923a73fd35eff2ed10624ac4e8b3dd65ca10ba29868084239f58ba4a6e
+size 9374418
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..69eb05abeae78735b00939848851088626ec5580
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:962941b1e0cfacbe93440e29249ae152400f37bb1e6a713339dbbfc86fd4fd1c
+size 3021489
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..caa6b54477f96cd183e5f9fbf498a156701c89d6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eac14220a69b9f5c63e005415c239595db6f67718258279be8a268737456ad10
+size 6997740
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2698796d28215bd9e988f8d3947a287937822ea4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c21b451363f9c033a21536cb17da31d311b82cfbce0827f763359a963e643477
+size 4291314
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4a6ed33da6daf2bc31d5851002267981ff1ef4f2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bf54ba7bf324c50b5407e92c9311785a5d1848910983308294b6d3f3cd06833
+size 5082056
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..04062f84d29160305bf280540655172e3ba3510d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daee09f4680aeaa433e2860fdf2aa0c1eabbcbc6e5cf14e8a9ecf60d554f14eb
+size 11742204
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..090a471fec5b7c48b5777ab0ebad570b0c076c1b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f55bdd99c54e8ab39f47501be06aae62c5234aa50d4a2546497ec709b25a06ed
+size 6665593
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7842c2f116e2025f9ea14b36f2a6dc729bd65ed2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2319268378fb538986329c0fc299cddad711c2186f04452ad6deb13a50b7cb4
+size 2868877
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ab5ff64cd81e81d42e14a15b142604793c4bc898
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f95d895f57a91d111ed8f1647557ba16fffccdde5b3856ed0e41d50c13e791a
+size 10394906
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8c14ba4bbc5dc850a222154060a8fdb0d23a418f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:272560cdb4026df25135be0d4ff54fa06b04ba901898cfd58e67b5ba1044915e
+size 7490157
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fab5917ab61b4278f87b25440e778aa7cb7c6e08
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3431ba3cd40ce4d4841838698c412bc231512c94926803436394c87f8d02d582
+size 9773029
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..992e1c62081add8e209cd09fb083334810a77c66
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2848bf7a8c36e65fbc958b1187de8802dca7b67ecc08efe7525786856e26ff3f
+size 11821776
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9cffdc873fbeeafaf6a62687c6f735af59326504
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f581617446a9cf7ef8636c202dbcd5ece9dc012a302dcc829df68e4328876609
+size 14076798
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..97478b924d3c5d8c812c980b397a03e3c8562585
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb69dd93f3d8c13244957fe8a57134828740bb3f354859e755d8b7d13879e424
+size 2782418
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f40b62d6d817b367f3fc9c6f93869e1cbf33acbb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d25fa360175d51c9e44758cace3154aa6bcdf94ecee58ad8493fdc1db56ca603
+size 9984082
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..011cc322683b66c15f25f66abb54311d34186010
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6d4e64c4427cf6d8a069f91bda3b558d4b9d4de759006ebcc8aae3d214a823
+size 7272356
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1cdf077e072192daff56cc4fdf31d35869aac262
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac04e54b410bc5f98adbcaf2e569dba42ced9a5e99ea1d5be55c0b53f2974054
+size 9532845
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b5f7640efb8524e924d30347f0f40f50c9232e84
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a608f47f11e57a23072695734a64403051a3b21075edce50555a042a895b52b4
+size 11574154
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..beffcc7a573dcf27fc3ad35b2edaa3a63f21b5f9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac270f0c35f6de81c74a503db28513830b8f0faf71dd0e79ece6647601ea450
+size 13794625
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1c40411f02965cc1f8b784d3255eec7fbf93166f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9b50f724c475bf44ac63819fbca5e2952a39500fbd0f2e4ac82194809da112b
+size 2823565
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..00f3211061e91ed1302159290cb7587f07001c03
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:041c5c890efecb7384976d67be01dab3c5ee1e6572dadbfa3aff712532f6529a
+size 10197020
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c360a3c74b0e8f1b04300f911ab9414d5080b686
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b26aa3202a0e8d309aa7e7de8ee56ac89778e18d53b61e64740f2011515a83c
+size 7372717
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..73b3b7b4d949cc17710f701f3ec32cb5f2cc62f3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f06389049f6b4f4dd61907e7b605e59c1ebc11c9c9efc7c804c66dbb4c34d79
+size 9641226
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c6ebc2f758de9a0b51aab76d6861f6481fa34c15
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:837535a6da1c2b31431c4a782c530650fb33cec60b23777a9dc0bba03e751cec
+size 11672382
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cee625ecae13d2974408fb3e08ff4fdd1100d735
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b3499091825f8e30459466c0ad84ee51e379b8a2a665e2bd7340c70a646136c
+size 13897640
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..86cce88841220b16c792a7eb9a7fd24354486f71
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14a74fca3645fe37d0954defc481df3505e831bd5ef61bc5d6085054b6c697c6
+size 2810592
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..43a86d8ceea94d8e1778b4e7ea3caeba3c6a1030
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bd934e127d9124b469d1d4d05be327fa55f5fc2fb7343231732e74449183164
+size 5080035
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f36902c002431adca7aa3e76c0a6e03a92d852a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f4960c7259d929d669ba0f0b90721249ff02a332e9a3f669d1e525d35a7320c
+size 7354983
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fe9faaf69b288b50b54e3739b5c2b094afd2185e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a7ed46de9e53dcc79b9f1f681f8d8e272b1f45f99b633e91d97a468b29c1e37
+size 9623893
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ddaa398c28c0dc9c86061dde0f5940534489f1b8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14345699972bdf8645d13574a318d8ece3afe6bd99eb182528b4ddf8d0a10b61
+size 23311946
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2bc37667f0cbb95582cbc21bedb2eb15b9068e36
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25cfdb691a50c0cad01d3fa35dac65baf7de6a1175f82703a1aad33a8cf796f7
+size 13882414
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..95b32d516c38182ec06744fe4ec3ce6e93ad0003
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ccfafdef659d08cff26b727eda0465c376de2d94a35e4dfabb8d22dd6022640
+size 2865046
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f9d07aef66b5f6bb71a1fb8a330a8b9189e1f4a8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9685a47ca100ccce5149be2a1933501ae7b432f64d023fe55abe018b54ff92f5
+size 10290844
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3ffac236fad7930e29def02b3ee6718d449be977
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d202a1b3e5f30832a76a2cd3d3632e461899fd314f596d029e4a643a4270e515
+size 7444556
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0a998288ea215c67ec4275dcecf07b953691cf30
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17ba0aa7d7708772a3d5dc48d8899ba7b0539b4f9adc88771a3e8145423333f9
+size 9726006
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e9e4737dea13c65aa8be7140fee4cdf3a77543c1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0839ffb71593832fdaf33d4256dbc407585dcd82484cf72c7e54b81167ff2e11
+size 11770975
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fb620a0cbe16635c006e553838d72621075317dd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6aa7e8e9549382387365cdacb9ac41c1be775a9da824e3f73f6ecca7872bda3
+size 14018654
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5aaef6ea318582906e0dd25222dee59f4f6ad625
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc993aaefdc006f8ad9432dc7b417870a6138b138a27f1af1bca314242774eaf
+size 3599760
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2bf6a19497d7919d0e398067a6a43f1c69ab9d04
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc0af33f3c47cd7bbce4b97339daa7ecad1f2abd7bf68e4c929466f5e3075338
+size 8331362
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1bda0198b8c82107d35e814c7b9fe61cc9ed89d6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2f7e9e4ac8dbce158ddf76884ec96066b56ad07c137833d99eea12aa61e216a
+size 4712263
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7be3d21d5c7a5d557ac17eafeb215e2908764a5f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb0c6e4867fc6dfecb2029cee26d848f3fd052829d0830083067917bac77cf56
+size 10615218
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..81d4d52bc85e16ca288217bec08505912f447bc4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7041062b4f791a327ae7bd726cc1cbfd62e3f3b53997743b5f31d068b97f78fe
+size 11863874
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f738d8600d88b9ea8d25f2f23504a5e98bc4aeed
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_Correct-the-solution_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25c774519bb06dcdc6a90c54ae401f802ebd15f71543d556069803654968dff8
+size 6564504
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..071b65c7bd6687278ae41b992e408024ad3042cd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a41e4ea34bfe50c8d4edbd8427038fd081560b351c2d564c41d2bd9a98edb610
+size 2115935
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..87515537248af6bd2c0b89e966164a13022fe61e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a915b951cb01fb17c8c60b6c24f22b1b918eb6b89cf16c3b0f6022ca787481
+size 2900909
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3bebff82237f651cc0e89ba5cb4c8a415f38eeba
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1bccd4aaa5884f38dd9a54fbca58f0660beda77194edb9af12dd7b0cbe2c511
+size 3683911
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e214c843c1689f3f8f565969a87b9d23aceb4794
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8dfa353d53bf00e2fe37530c0b07f6adf7a14141a238c0368befe443a5a5e53
+size 4458372
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bd9797e6504bc50c3d09108c8d774b2a6b33f8d4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db0f287e31a9495cfde302feb8108326cb85101abcc67d525b4fb3f1a46189ba
+size 5254516
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ffbe1f9cde82ab10b264d1a55f08caf19b4004a0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:237520c45c997402d16abfcfcc7dddde55b2d6ec7b7c58642ec25cfdfbad8eea
+size 6040677
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ef3d68c14f78ac8ce9c548faec0059e5c3243657
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c93f09be6ae63f4057329bf8005bb527cb5587e88e15f5f69d540b2fdbf2e9
+size 2821393
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6eda178915a0aad49b30be2e9b47d3661a875a30
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92c03b626d42ec209f10cc8c3b02bcd6bd820ca04be1ad432728e622a4c12fa7
+size 4395320
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..44495aafe18ac4a1cf8d8a6c0aa1c304323764eb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3588f642137097c85d1dde09f83c36182dd99d10bb3af00ee492b802a4e9721f
+size 1979130
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fa186d614a57a01e01f67253f3e2431f4989ee03
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b22be059c12bfc6efbfbbd0c45208b89d84550c8037a9ba23f4f83606503df
+size 8705536
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..74cdba3ddd1e8aeb010dcf15758a3d411210bd40
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2d8bc33273a4428419b223f70a417881ad2cb0bbca88977297b2817cdad3bd6
+size 4906692
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fb1253dc32c13f542cee309b89d3e0239c9e5e6a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_no-prompt-needed_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a0aa3b58236f20390da5fce3dfea5527bf3f1decd372d27741b26077ce53db
+size 2710753
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..db38eeae18cb274507f443e92f90086d85bf6131
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fe6d1ec7b37bae5b3662ab78623e555447052a034a47a8948c3965986ff33ce
+size 1864129
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..429e464188a5355b3bff68b69d582c81644ec8c8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8748e70606949c1c979b5dbfc9039750d1f50cec4ae647c97b4544208538f968
+size 5114406
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e089dbad2a47f585fe45c1975a51a9eebb06e53a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb8522c02dab42dd932c554d128cde6b4772f68b238aab7953da6fbd9178354b
+size 3248305
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..327051bb47ba99bd40b490a58dcb37f47caf3b88
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93786e68f9a19c4d7ef594b7c765083dc44eef21d276d9020afc6ab389664937
+size 3930866
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0a7f55867bafde03b18ce57214d54b8d24b0285f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e35ec51ac68fa17f796252f535295c84b79378e0311dc03aec9585595557f15c
+size 9270220
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d72aded2a8362fa0d24d0a8e6c57066c80ce3a93
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:633d1d7ae670c8e53561995249cfc7afc72fac4dccc48bf7427ca78894ff2465
+size 5329371
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..228f80ba0a3404b3e0dd56bf98f930eaa09ddaf4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cd55217fd616f785831503911b8264922c13942f936dad464deabfe0c058c1d
+size 2256879
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9f2e3884254b96ab431c44c25529d9a4a308c63d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ab1928a3e1f7ea677200f0c9222c4b3ac30512bf8dfe83efc444b6b7336e146
+size 3016802
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4499065c69993b0c48d7daea8224fdb8fef22845
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3ee5df45da21c92c5f7f63d3b0ac134af6792a67f0b839f97f159273487631b
+size 3773050
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bbeef0557c9ab9406f2a1015dee479a3c2786112
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7b03c45d1d3559d7f11059673dd20c6cb729753e4fb105496c1972a60eddeba
+size 4517638
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e77d6799573698f35d3740e17f2455c642f2e4b5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e760a2a82aeaffeea6f9f3f912019ebfec8e6ad54cb0fa077c3d2d26d8ef40f1
+size 10585606
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c0b1a6d12774645999fd74cfe6d19fc0d0970692
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b02a1f7cec2e63f0a7cc0534806300d1759a46c30ba2020610da3c88837b691
+size 6054070
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..74c37186ba86004202e17bf03c7d2691f0cb8c55
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3f0277c8fa121083f67b48f0036f164d8d92f9dc06fd26a77386d436eb13603
+size 639440
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..aae989a52c73f495c3a69c99939d187b52fd37f9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb90070a3d5e37216a4d7ac11f511d7dae5a41eb4c5b9a02160b38d801eb1a7d
+size 754492
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..40b6729dc1cbc9be196d5ef4897815896b794a04
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8225f3567f43de5bfaf3b2e7c2f410cd98dee161dd7e24a98d8d4e64d6fc1b01
+size 870659
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..aa046eb8992504f62adb83e76ff96da6a6be3759
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d133dcf4b93438b7b82c9b13bc7ff356ae39aaeb252543476324e977cf826f5c
+size 985104
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..58c88bd7449c6bf5b6c95b9c9ec72d1ac77e76b9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8988fd4efaaa940f35ce19378543fafd40d4228db0269c14cb862eb75c3d9b5
+size 1097802
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0a320e323e04f14d2a83e24f3df161442ada1143
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db0ec03aa37458a137461234b9572bd2fe8b14489d3ba977bc75bc2f1905c64
+size 1213046
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9c25214497eaa5fe00aa636912c2ae6ac35c3e50
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6bf1c8acb4a644e28f16c3731a6ae4ad8e87dc9f242d95534735ed8585f6f4c
+size 1182068
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9c3fb2cbe51c678cb023994d81c7bbd013e322a0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c5e56598b9914f57ed3165ca40a136b37e765ead618ec9e1fa073a6ada1b87
+size 1778876
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fcc2a8ef39923e42bd9debb989ce508436302ef8
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9104bb1d1735db69015a10f3c63c8a87f0543689777ac6e5ab013c23ed1b95e
+size 2388047
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3d9893276adc4b340313a6ba553017d4a4c3b20b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0432b0550877d6ec3e45f71b3b2830634d8c82982dfa8ef7a08c90fa5d1dc99b
+size 2973046
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3617e3027619945d690cec4786128a59fe21ac35
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9a1e9b75663dca80f16fee7f8a315d70317149aed635db76f00b1774924ca6b
+size 3555436
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7400df2fa11b383a7567e805a7f9cf31ef4f3067
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Direct-Question_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c623020f21fc1cb7057598fc993b769924d943af0d36f2ac08e64a1faa94c6e
+size 4144288
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a577cc0d297506a97e25c0107c4aae5692cd9b76
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:136d5492d0739b0fb4c4bd869abd618ea9955d3c88ac41340febc6b71ab80e79
+size 1328790
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..492e9ca8aa7b494bba85813bb9a41d63a04de14c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c10ff04d2da5b4f5f4538fe105df1bfb7a12d9635b8475348583ff686e0635b
+size 1524208
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..430cab2f5af0d788c3b89320eb76c3eb6a6b2371
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:727b7f2166d1fe3ce63da6486516a2459bb7c7a3c5886ccccb2caa289b3ab8ea
+size 1720226
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5a58b02296789ffc1bfaaf2bfe9189c76612d6c6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1822358c55f9fd908fb00ae4769f387373d895c38cde249ff2e6e71730db227e
+size 1915295
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d36b961629c9440be5937b148e55a64e2a38f377
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afec9c8d06f8f556a31f1f6758faa73ff6e512313e7021b6e6518bca59efb4e4
+size 4214180
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..639dd04d972ba9a05ada1abee50ecfb3b28238e4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91a1595a79324d94fb580708efb0c91a5fae57ff36b7d38628ddb147d260e5eb
+size 2300435
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2c684c1433744867b514c4016a35fe7350b961fa
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d684e5958a951d687639f57acaf3389e2bea4ff7794d80ecd2430e71f61c7c99
+size 1934893
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3a93c115c19f777c00aa4041d65326ea13d3ae37
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed65d2f92c100c41b18a0d365d8fd361fd07ed61729e1f71b488b18e55fde588
+size 5272296
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9b9b307bea0a2b224eb5ac836187784f5a1b16fd
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f258c55f9f24eb43347b116704e81e9e153c11e65c60a94ef5c6279f5da59173
+size 3349136
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d59e34983da1cb2ca706562e9897345b19c46208
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf576c99b6096ae01fde110747324e3413c43fdd8a2159da179be4da8ee646cc
+size 4038707
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..99e478128e82773a71863d5a2114635f28b56e8d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea233ed65004c243d8bf2d464105458fa1b423068b29be111188b9d08c773836
+size 4724101
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8ded1953fe64875432d55593d7a5c15120ec4ee2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af2e6d4043cd47df1ab0861561bc71daaafc4ffcefcdaae4aa1b8ffc97eccea3
+size 5415228
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c2a9e471e14bcbd0c5a039d99dda59ce2581db0b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e188619bfdff0675c1abd16a8338e53d07d11dacc0824c16d5b615b8c22d769b
+size 1869837
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..77acb87a3d0720791304bd5cf5086a176f8a9e68
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b39f0a787e6fedc4a1636275fc9286908ef346248e5a465fad20b49726330456
+size 5090146
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2396938bfa00407198f68c872899b85944ab9856
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb2c67db25fbf7354c5d38f3cff7cb8300f0c96570dae3902198455084c2ec3f
+size 3232146
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..67cf11c8d6639c7ce3b0f68354e97f7a3ae9883c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3be2502bcaca382df3c0b0df31042328ae31c74b4dc322d089d4cb422d6ac1ef
+size 3895713
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ea471f9972b3613368e1226fe36668ef3656df41
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72caabcc185a9632ee528c0f7898535adfb10f1b3ffe4fb186eae2d8fdee4154
+size 9110214
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0edd14aa6cb5903001e0afeb2d3a51727b3604e4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_sciq_Multiple-Choice_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef572f6241470344cfcbe999e4fe662a52b653bda70a3e09c355ea76e5f894db
+size 5220220
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..84c84aca8c96cea6381825cdea5be184d4aa8f89
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb87de8eb0b131e6eb52011d2c1d71103c40527ec8b52814a3dd7906b78b6d63
+size 2212575
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..123d095c5b45c90a21720211145034a37098da7d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5176991b346e053cc33f59306b612cba6b63737042d2b01ddfd3a67cc55f3f75
+size 2973133
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6a94f5464c61f01ccac84820f517651294d1589a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a26e23a9786efb40cc1c0602dc7fd69c02194a36fde906a94e8ba627ce0f09f
+size 3724950
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d988c98a724c7fdbfd160848849a643185efc3bb
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1efa4ab60a4ab0c07805d0c1143c43692e2d5fd2027bebaabaf40f89ccf01d62
+size 4476043
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..25ea7d4f735fb2eddf8d147a25804c6f157f1136
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d11952b1dd8d919287981a8c3fa6688efcb0eba8ebf2027ef0bbd037eb091c76
+size 5228025
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a443cda34cda57429312ebb4dadbfd384f25dba4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da0a652a647256fa310dfce48bdc309328e8a6eef38e14b0cb712bb8c5b65302
+size 5978592
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..87be2abb7915b9914506c7af37fd87d49cf3e339
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5a30e2e04f787dafa6a73710ccaca774ba1a8f83df4c07ea848233c0fb24bda
+size 2360385
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ba933abebe34567b0bc262cedbbc76d1451fb118
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bf1794cd73e38f784b77dc6d5f2859efab0dace2acb1271df9927e702ab7047
+size 3191664
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..708efa27517189cab7a1a3a59e18b7f23781833a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef5b54ba324b1e759311818b6203dc61a4fb3f7a6652975411be7b34ab377655
+size 4014786
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..59e07341f75efc64b8ada02b85cf303d8f3f8612
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ddfee724b058115533b92c2b7769cbc5124b8dc8e0636f36d3df14a34547ed6
+size 4837184
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2552935f0dca1a0bc03a1489d19ee2149cd0ef93
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ee74dea5ab9f13d50fb18ca4ed7a7de21ede2ed1a428c03abc54ff395ebf227
+size 5660073
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..335f9c47e8aad679e96f28d54678cce6b8081425
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7190e22242d26ea7e0977c2c762bb07c243ac48799d7ac9f1be430f1af063cad
+size 6481752
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..90a671119b5e83f9995153ac91890249b12bab30
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1ce66815ee9263fb0ce989255fd92d661cae1fa1e4e5855695208418d8ceb4d
+size 1877681
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..87eaed65446351a3585468efd6a5d62d905c061d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56fc03e0d41e873711b73f16903bd119b1208c8339aa61b704414c240533323c
+size 2433519
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..77144920322dcbc1cb196978d111feca05fc41af
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5962bb279b1256fe650256ba594dc8269dddb915d6f70cf84517aad799b10cf5
+size 2983949
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9b3e4106d18574cac980c172561280fcd093b085
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68ec6b7450c9d39a150c1d57d93531eda0137db16797c95acf0f52a1f00f5c13
+size 3532385
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3f290de8522478a8b77260e61a79a3bc4a1823e0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc797067f058683a51960bed0c568e7324c7fc0292541324576d35b61cc68655
+size 8161392
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..12127ee6eca77ff76e865ba60d0d24e1be8e7035
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d3ed2f43d1d94971d487530c97954c082ec0c2f303de1cef2c604fffc133530
+size 4627673
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3a49ea273a186e37424aaf97745813e8d8e615f3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2bca2a13a89f84a61c9c53464c2cbc4665e1d192745bed528dd24be14842e46
+size 2369512
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5f09a4875f7fb3fedd99920ce2ea15b80633f1c5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f0c4fe5fea3b20a412a3bbcbb2789bb2f20b8f23f05d54a59d2d3d5336b4ce5
+size 3208334
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..700fc3f4589dbfd0fdf3ea06b32d21af5979634d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac7e1105bb2cde87668732c63527dd079d1f27a6389edd9de149120742ed124
+size 4038809
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a0d69ddc049949e6159e0ddf4a2dd19033aeafd3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09e756c2cf843fb5095231b868f0379435df9219dea40d807fd51ae88939f680
+size 4868610
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..87ec9c878b72587837efedd99cad3e84ed3e6188
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:662df36ae40712f5215c1aff6fab9e9a182273d1e7ff91fbc169d3d714c04a0f
+size 5699056
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9433aaadc79963c3c9eebf29f5eb11a1729c0996
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fdceac6299ca5b7dc61e4f6a8a0be6dd868291e437aed541e433506c54872d6
+size 6528411
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..eb712de297debf75a8eef6aa9b4d46e6ed059ac7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daa1d35904071aa01d39956c081efbe0724d8c406346c820df193f95fb07c416
+size 2343462
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f58ab89f02ec9e3556b71e213d97b2ae7d315474
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c92639101244cfb38994ab40727b7ebceffc8d57d457d400c9edc2dd6637b9a
+size 3155723
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..03503dd7cc8516bc9c1f5b6310cbfcbd8a527092
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cffae0370ed640cb7a4be40c6001ecd0bef460b59e4db2504e417d64a858c0c
+size 3960144
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8eb6c545781c8b7e9308207d6fa7a69f195e9fe6
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e724c1863643c43bb299612866232d10802585f0bd400e33fcfb39935d06cc9e
+size 4763931
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..652e432a07c1e39a85b90bae087cd629967072a7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a88a5c160c0b926e2b27b0007ebbbf840d63b3c1b32051f7efb2d73d789b44
+size 11136036
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e4a760f9bfd5e6a4c310ea5c9159c3ca79891525
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38d4201607390183a74d7453dc131aa5c7333efe892dce812dd124009ca8a353
+size 6370986
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d5d8ec89e8879a36004f12bfc0166a99fa7850a2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b73791b5cd8d5525636d5b9801facec76c363388f460d8925be711a1ee4bbd27
+size 250481
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8bf5db0b11422c0a33dcdbec53479aa1e059b609
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:befffbae478663b0a269ee27e05d5a5580d59326cbc5f806e6c564b5aa029a60
+size 351379
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7ac245115142fc473528cc54df9d90750b9bd964
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f979be5ec38b7111ac601e38c0bb0b34f18344bd5175a6634616c5501e55bbf0
+size 449685
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..71d7416632fdc9c6c1a9690df5308ef1888c8a69
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e69446ad423beaef324652baeba117a3d2f7d794cfef803f871b961dbe1fdd8
+size 553066
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ce5d7575bd8768f313bcc9093baaba79ebdf5a9a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27cf6e99af218ab993831828aba19189b6c2c9dd45420654d15bf4cdc5524982
+size 651670
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bad14608bdec830459930f7554c5d6d8e4cff123
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:903bf72fee08d526afc8285757e9cb00132f65358569796bd985d6b8b776aa9a
+size 1495168
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a9e74b67b269ff55afa65f3a64df56e1478e7090
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:287e2f74b99e7226ed280c9e3f7b03eac0c4628939d5e5f5f8ccac0a77a87805
+size 293135
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..77c0f8d480bf954fb578d1043459d40a43a0c9a9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0a06f820278d561016b192bc8075a3b7314a10668b22209a99910f30591f346
+size 415394
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f78771c6406708a23b971f6f91888015031052e7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1a0266d607cb878771b7fa533511694db01cffc49501b138f4d5a792c71be7e
+size 535345
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..461adca4a28a450b31d2a8381942f7e3ea7fd1b2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14c094f1421f2109be65fee21e489aa2105213a993e2ffcaf1f9fb801c7e4fa2
+size 660321
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f28ed3a7caff93bdf0826eb784b3a391b3e02ba2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd40cbe24f568fac6715f36efe9793a1f4222dc643b6163f2d5ee3ede5c9f7e2
+size 780564
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..af68243e9cfc2a7fc664082dba609ab4df1d4275
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7fdf3aeb33fe82da78c076f4c97b1d19cafadfbad58f62044c91a3ae4d176e1
+size 1796126
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b93f4051c8d4abadd0c52c5fa3d00cec5a5dbf85
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:448ad7b969251c428bf26a1092667b52764987431c510dee8789e623b727328e
+size 258280
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d80d8d080d5486448378e2b6434d7ed525fd4ca4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe60cdc60c9af2908f329c8f45c8791a99793f5ad384ea4cda56de95a9fe47de
+size 363041
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..db33634bda44a226d92f68dbd8fdd8be38185e1c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34f40e0c66fc91637787df954a7cb27f1715640ac7dccfeb3c35d56e189e10fe
+size 465253
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0475a0aced6cb2a750093f145787ada4ccf9a39d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c87c4c5ceb086b14866b35e84d3092d37ecd80dfa1c46a484ea89839c7dd6d4b
+size 572480
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b43b4f37860978de678044e7b9da539e15c9bbe2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4467ed159951e571cde0b99383d3c29552b6791fe1400e90c7303f95db48d89
+size 674990
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4022af59e6de7f093829c59f60656410d24f639c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bd57194bf35939f00a2b44e763905c36f2734095516a2f81ab05901cbaa8809
+size 774754
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9e74656e1e077c5d8ae79870783417c3a7f5b30e
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e7b78d98d62496a3c82c4152fe79a63b48d0b1801483523595965d0ea245257
+size 261242
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2689d3318836b5dcbbc30cb37e82f9d5586fba7b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61227a1214183ae12588455ab074a34a19e1f8ad7f561d8cd1915462b3ca4d5d
+size 367750
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..262a690c76b76b5d3ddec9e7a3bd810bec5e8069
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3240a2259e850380e8427eb73a5100bfdd3397aab709c26f6d9f5541a9a99430
+size 471922
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..257d51aa790a637cb2edb40513f00d6ebb72a524
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad8cc20002ede2587d6e911dec56dd76add52ca328dc6af971819de550af3e68
+size 581090
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ff3d18b144ee65b81712ff96228c396f460cc83c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab2d9d672bb69809b8ee9e32d9900cfc2d1053d8dd3da3899d073d99de6a985d
+size 685535
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..98e5428f95dd9ba627fa2b070d10b9a960616afa
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87185d10e01d2a6dfc6933f671f68418e8e2f541625fbf185684fa6d8b337858
+size 787257
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c64f1d676039ac4667c883c0732840e08b1f7027
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6969d485d47f84344809887ec2c727ec9eb1f7c2f31c95a29c1d31037190a0b1
+size 262365
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..351a660da50ad873c9b3e0569cc06ad15494b66f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d98661700638905f465e2a68a139d716fec832e8af793801dc2503f2643128f
+size 369689
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b09a72892273dd70e0397f4685b862527f08bcd5
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d11883f510594378290692a310db168d3a5d1b905914acac7ac98b451751b8b2
+size 474677
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c3f898513940681454a451f54e4a0eef38dd0aee
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c99671868d8ef7cddaef74c7d54cfb7c0ebc61cd69bcb89ea424315a7b19f863
+size 584694
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1a5610621d8cd4cafa3eb08cdc5526f9da5af8f3
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:173d4f53d2cf01aaffab5bfd1f25b71b54474aa8b90d33bb365a5e57f7e1a9f0
+size 689968
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5aad02604e8a53be6efb50a9b9c878fa878dccda
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_superglue_rte_should-assume_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:498cbed8245fadfbba54589d32a7d8f7555a703970bfc5f715c22a3f395245e3
+size 1585060
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a367ee9bb32c4aaafcef000103fc225a0f6b6b18
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7324458b9d08a7087e1108f54be02395d05fb18f213c2978980432cb04e02712
+size 1039047
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7e640a746a292ba06fa2e696192293241b2bb976
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:808d74144e971ec7a279905a7e3256a0eb5eb6684e86b048afd919e3e534f4db
+size 1299935
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..28b0413f642c6260e142cc807b11a40335716140
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8259d114479c155656af556a53bf1b00330f2dbbf9bf88db5555b9dc68ed2889
+size 1560966
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e7a0f2b4b4fae91971d637c47c746a4a68fe025d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe34eef0bec1210a4cdd34dc62bc198bef1730e9d67ed47bdae5045c65511991
+size 1822458
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fe0690477829e0a666f2949fdc8889c227a34224
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fece369f1641f40a4397af94adc5e523bd1c0d549572312f419610f02124c0da
+size 2082729
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b023abb2962cdddb42e8d6e10a7b1facd55d3840
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_Replace_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73171d0bfd491793dcca615858803e85010690f9b91d80ae3f59781f3161c22e
+size 2343414
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0df8b70e2149ea9556ce555bbe0d09f8959da77d
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7394f6c58b430d3f24792f8866102a3f185c895814550f7c98ada5d094cbb16
+size 948114
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..44fce27da583c1b0f0b1a67115437a9afd0efdef
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43041022213e65239a39b8e714c6f19f033202871441e0802f1f429e6f4fee8d
+size 1180688
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9af61b425c17d6b988edecaba917a56d7b9521ea
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4030fe0a7f3c98c1b6c40e32271b8259e4d4b175be198508a28e537e424300
+size 1413236
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d182c9a886a4b435a28114f7475ef066fce1dd0a
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c09e50407dcd0b71d2749e61e1144479d08dd8df5a7b24455c845e6ac318263
+size 1646079
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3d0dba32a13981724d92e4a4d2de3db2d9d7e9c1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3487ca51990a0544a2652f4b52a96fe73a9eb4868de266d66ad2009a14ac9ca6
+size 1878009
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..acede3637c323be1111ef9d1d20f5f339af0264c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_True-or-False_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20a58defc5b903e0a2edbc97edb953b35dc42546260bff82886151324d022ae2
+size 2110064
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..627fbf17fc1b1c3b47bbcbdf521a7794d98b6849
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e28f61a7dd90619a818819444caa22ca133818c86cf7fc318c4e884bd52959b
+size 1009951
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f891a56813b814dd4df793365a9cddff34f3d64c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddbeca105de1ad974c6d060978acc1138d3a62cbb4cbbd75fae0e8b84ecdde1f
+size 1242880
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..561d469cc8525c8e2a20c9740c35ee787c37e4c4
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:606d84f5fa92aaecca1237f5b24f9b3982a73e13f27ecde3df81c90f9c40aa0c
+size 1476150
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f1e1c74a8f96568956d8fa1c5df438723f11232f
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8fe4d5a200b5cefb478dd8a5237276dd3fb7536d6542aaf8a061c48a0ccd556
+size 1709719
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7e00da7848d97ec6ee98c33ef708ea2d17471829
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da4d79e1a2142a7d78e25a90566a57813196d7ad8050f46dc2a7827cceec8b4
+size 1942062
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fa9007ac59fb7dccf0f2f0ef85e3b2f563aefc1c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ebccc4955c1beda768478f4d638955beaae23ee2a7892efacc60841d349e13
+size 2174945
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..44a3edfc15fcde160beaaecec049f57d4f8ca1e1
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f37aad09c8e2e62c8c2f986499d508d38cbc9b5d351eb32a84b30e1a2de77c
+size 969191
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5043dfbc41422fe1a08e7160322747086caf3187
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f1be122354d00fb86f68c334428b5db0354a87e68e035e8e5924cb5eeb02da7
+size 1204734
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ac550cd23588a27416c8e8ed75fe0857ed41bcd9
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be87a784837ea4d6d0d08be3b1a2a60f7ee6cc0e3599c24d55354e169ddb7cf
+size 1440547
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..71cdcbc18a29e3cd1f82fe1342f3f84b82a27ae0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f4f6ad112a29d628f1b624f449ffc1ea790317ea03ef5806ee22a06f4335b56
+size 1676657
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dc25bbd65c3aa603b1f0e9325eae1b2c6e4fc45c
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72eb72e72101940016dc69949f355afaf8d0eded9f7779cebb31073794cab4bc
+size 1911620
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..490bc7b3f8626f20ac31469332a39b8560a7ea27
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_stand-for_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:070580c7cfc6159c2bec97a038e21105ea8f70136a5b0f3ea7dddd61f7eb4f87
+size 2146985
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_0.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_0.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..883dd7ae0745560c5e9f001f0d44d1a33af4bfd2
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_0.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:240ed9517d474313360e9708648a4b49e1a0df3c93e526292d098b883c67b4c6
+size 1016303
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_1.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_1.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0fb0214b8f783597d007732af984aef93b057de7
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_1.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e62f7b3c72fc09f6d0f4f7c3ff94eee21533031ebc55829953aec0e9451d8d
+size 1256962
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_2.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_2.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bbc42983a9e9a6aad490356090da1bb36354b827
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_2.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf5ab68510bde4628ce45070dd73bb5e64c228517684f75b6abebab818fddd66
+size 1497719
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_3.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_3.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5238250c61209cd55a100da7489ab101421e5d2b
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_3.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13674f108e37c5606b7bab63ea5402683a56955362ae55444ebffa4264fb1ae9
+size 1738903
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_4.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_4.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a16604f97026c8dfc3dffc797fba2b04be9a5d45
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_4.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9a0f092445e0ae2258aca89a6c0717fdaab88e9c7548d413441cf3526ac3be9
+size 3957694
diff --git a/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_5.jsonl b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_5.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..04215f5ec23948923fdcd97053f3ab57bc65acb0
--- /dev/null
+++ b/1b11b51b5/eval/examples.lm1-1b1-1b5_winogrande_underscore-refer-to_5.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f45a87a99bd4d1d79d09d6ddb34b8d15c8626d046d118509647ce0d47051bc1
+size 2219308
diff --git a/1b11b51b5/eval/merged.csv b/1b11b51b5/eval/merged.csv
new file mode 100644
index 0000000000000000000000000000000000000000..29a2555c03ac0b2c64443dfeae738eed1b360c03
--- /dev/null
+++ b/1b11b51b5/eval/merged.csv
@@ -0,0 +1,587 @@
+dataset,fewshots,prompt,metric,value
+anli_r1,0,GPT-3 style,acc,0.323
+anli_r1,0,MNLI crowdsource,acc,0.321
+anli_r1,0,can we infer,acc,0.321
+anli_r1,0,guaranteed/possible/impossible,acc,0.318
+anli_r1,0,justified in saying,acc,0.32
+anli_r1,0,median,accuracy,0.321
+anli_r1,1,GPT-3 style,acc,0.33
+anli_r1,1,MNLI crowdsource,acc,0.333
+anli_r1,1,can we infer,acc,0.33
+anli_r1,1,guaranteed/possible/impossible,acc,0.333
+anli_r1,1,justified in saying,acc,0.327
+anli_r1,1,median,accuracy,0.33
+anli_r1,2,GPT-3 style,acc,0.354
+anli_r1,2,MNLI crowdsource,acc,0.338
+anli_r1,2,can we infer,acc,0.342
+anli_r1,2,guaranteed/possible/impossible,acc,0.336
+anli_r1,2,justified in saying,acc,0.346
+anli_r1,2,median,accuracy,0.342
+anli_r1,3,GPT-3 style,acc,0.347
+anli_r1,3,MNLI crowdsource,acc,0.373
+anli_r1,3,can we infer,acc,0.349
+anli_r1,3,guaranteed/possible/impossible,acc,0.339
+anli_r1,3,justified in saying,acc,0.351
+anli_r1,3,median,accuracy,0.349
+anli_r1,4,GPT-3 style,acc,0.337
+anli_r1,4,MNLI crowdsource,acc,0.346
+anli_r1,4,can we infer,acc,0.341
+anli_r1,4,guaranteed/possible/impossible,acc,0.319
+anli_r1,4,justified in saying,acc,0.342
+anli_r1,4,median,accuracy,0.341
+anli_r1,5,GPT-3 style,acc,0.344
+anli_r1,5,MNLI crowdsource,acc,0.338
+anli_r1,5,can we infer,acc,0.332
+anli_r1,5,guaranteed/possible/impossible,acc,0.324
+anli_r1,5,justified in saying,acc,0.339
+anli_r1,5,median,accuracy,0.338
+anli_r1,5,average,multiple,0.3368333333333333
+anli_r2,0,GPT-3 style,acc,0.32
+anli_r2,0,MNLI crowdsource,acc,0.336
+anli_r2,0,can we infer,acc,0.336
+anli_r2,0,guaranteed/possible/impossible,acc,0.348
+anli_r2,0,justified in saying,acc,0.311
+anli_r2,0,median,accuracy,0.336
+anli_r2,1,GPT-3 style,acc,0.318
+anli_r2,1,MNLI crowdsource,acc,0.315
+anli_r2,1,can we infer,acc,0.318
+anli_r2,1,guaranteed/possible/impossible,acc,0.315
+anli_r2,1,justified in saying,acc,0.32
+anli_r2,1,median,accuracy,0.318
+anli_r2,2,GPT-3 style,acc,0.329
+anli_r2,2,MNLI crowdsource,acc,0.321
+anli_r2,2,can we infer,acc,0.324
+anli_r2,2,guaranteed/possible/impossible,acc,0.32
+anli_r2,2,justified in saying,acc,0.311
+anli_r2,2,median,accuracy,0.321
+anli_r2,3,GPT-3 style,acc,0.325
+anli_r2,3,MNLI crowdsource,acc,0.325
+anli_r2,3,can we infer,acc,0.337
+anli_r2,3,guaranteed/possible/impossible,acc,0.326
+anli_r2,3,justified in saying,acc,0.333
+anli_r2,3,median,accuracy,0.326
+anli_r2,4,GPT-3 style,acc,0.336
+anli_r2,4,MNLI crowdsource,acc,0.325
+anli_r2,4,can we infer,acc,0.328
+anli_r2,4,guaranteed/possible/impossible,acc,0.325
+anli_r2,4,justified in saying,acc,0.329
+anli_r2,4,median,accuracy,0.328
+anli_r2,5,GPT-3 style,acc,0.333
+anli_r2,5,MNLI crowdsource,acc,0.303
+anli_r2,5,can we infer,acc,0.336
+anli_r2,5,guaranteed/possible/impossible,acc,0.317
+anli_r2,5,justified in saying,acc,0.328
+anli_r2,5,median,accuracy,0.328
+anli_r2,5,average,multiple,0.32616666666666666
+anli_r3,0,GPT-3 style,acc,0.30583333333333335
+anli_r3,0,MNLI crowdsource,acc,0.335
+anli_r3,0,can we infer,acc,0.3433333333333333
+anli_r3,0,guaranteed/possible/impossible,acc,0.32166666666666666
+anli_r3,0,justified in saying,acc,0.35
+anli_r3,0,median,accuracy,0.335
+anli_r3,1,GPT-3 style,acc,0.3408333333333333
+anli_r3,1,MNLI crowdsource,acc,0.33666666666666667
+anli_r3,1,can we infer,acc,0.3441666666666667
+anli_r3,1,guaranteed/possible/impossible,acc,0.33666666666666667
+anli_r3,1,justified in saying,acc,0.3466666666666667
+anli_r3,1,median,accuracy,0.3408333333333333
+anli_r3,2,GPT-3 style,acc,0.3416666666666667
+anli_r3,2,MNLI crowdsource,acc,0.3433333333333333
+anli_r3,2,can we infer,acc,0.335
+anli_r3,2,guaranteed/possible/impossible,acc,0.30916666666666665
+anli_r3,2,justified in saying,acc,0.3275
+anli_r3,2,median,accuracy,0.335
+anli_r3,3,GPT-3 style,acc,0.35333333333333333
+anli_r3,3,MNLI crowdsource,acc,0.325
+anli_r3,3,can we infer,acc,0.3375
+anli_r3,3,guaranteed/possible/impossible,acc,0.3125
+anli_r3,3,justified in saying,acc,0.3458333333333333
+anli_r3,3,median,accuracy,0.3375
+anli_r3,4,GPT-3 style,acc,0.3416666666666667
+anli_r3,4,MNLI crowdsource,acc,0.31416666666666665
+anli_r3,4,can we infer,acc,0.3358333333333333
+anli_r3,4,guaranteed/possible/impossible,acc,0.31166666666666665
+anli_r3,4,justified in saying,acc,0.33416666666666667
+anli_r3,4,median,accuracy,0.33416666666666667
+anli_r3,5,GPT-3 style,acc,0.3233333333333333
+anli_r3,5,MNLI crowdsource,acc,0.30916666666666665
+anli_r3,5,can we infer,acc,0.3258333333333333
+anli_r3,5,guaranteed/possible/impossible,acc,0.32416666666666666
+anli_r3,5,justified in saying,acc,0.3275
+anli_r3,5,median,accuracy,0.32416666666666666
+anli_r3,5,average,multiple,0.33444444444444443
+arc_easy,0,heres_a_problem,acc,0.23293515358361774
+arc_easy,0,i_am_hesitating,acc,0.20563139931740615
+arc_easy,0,multiple_choice,acc,0.2235494880546075
+arc_easy,0,pick_the_most_correct_option,acc,0.23464163822525597
+arc_easy,0,qa_options,acc,0.29208754208754206
+arc_easy,0,median,accuracy,0.23293515358361774
+arc_easy,1,heres_a_problem,acc,0.24061433447098976
+arc_easy,1,i_am_hesitating,acc,0.19795221843003413
+arc_easy,1,multiple_choice,acc,0.28535353535353536
+arc_easy,1,pick_the_most_correct_option,acc,0.24873737373737373
+arc_easy,1,qa_options,acc,0.2975589225589226
+arc_easy,1,median,accuracy,0.24873737373737373
+arc_easy,2,heres_a_problem,acc,0.2558922558922559
+arc_easy,2,i_am_hesitating,acc,0.29797979797979796
+arc_easy,2,multiple_choice,acc,0.28324915824915825
+arc_easy,2,pick_the_most_correct_option,acc,0.2474747474747475
+arc_easy,2,qa_options,acc,0.29545454545454547
+arc_easy,2,median,accuracy,0.28324915824915825
+arc_easy,3,heres_a_problem,acc,0.255050505050505
+arc_easy,3,i_am_hesitating,acc,0.28703703703703703
+arc_easy,3,multiple_choice,acc,0.29292929292929293
+arc_easy,3,pick_the_most_correct_option,acc,0.23293515358361774
+arc_easy,3,qa_options,acc,0.29124579124579125
+arc_easy,3,median,accuracy,0.28703703703703703
+arc_easy,4,heres_a_problem,acc,0.2363481228668942
+arc_easy,4,i_am_hesitating,acc,0.2030716723549488
+arc_easy,4,multiple_choice,acc,0.2946127946127946
+arc_easy,4,pick_the_most_correct_option,acc,0.24873737373737373
+arc_easy,4,qa_options,acc,0.3005050505050505
+arc_easy,4,median,accuracy,0.24873737373737373
+arc_easy,5,heres_a_problem,acc,0.23378839590443687
+arc_easy,5,i_am_hesitating,acc,0.19965870307167236
+arc_easy,5,multiple_choice,acc,0.23122866894197952
+arc_easy,5,pick_the_most_correct_option,acc,0.22866894197952217
+arc_easy,5,qa_options,acc,0.2098976109215017
+arc_easy,5,median,accuracy,0.22866894197952217
+arc_easy,5,average,multiple,0.2548941730540138
+boolq,0,GPT-3 Style,acc,0.422
+boolq,0,after_reading,acc,0.6236666666666667
+boolq,0,exercise,acc,0.6236666666666667
+boolq,0,valid_binary,acc,0.6226666666666667
+boolq,0,yes_no_question,acc,0.6243333333333333
+boolq,0,median,accuracy,0.6236666666666667
+boolq,1,GPT-3 Style,acc,0.539
+boolq,1,after_reading,acc,0.5426666666666666
+boolq,1,exercise,acc,0.546
+boolq,1,valid_binary,acc,0.5406666666666666
+boolq,1,yes_no_question,acc,0.547
+boolq,1,median,accuracy,0.5426666666666666
+boolq,2,GPT-3 Style,acc,0.5253333333333333
+boolq,2,after_reading,acc,0.5673333333333334
+boolq,2,exercise,acc,0.5633333333333334
+boolq,2,valid_binary,acc,0.545
+boolq,2,yes_no_question,acc,0.591
+boolq,2,median,accuracy,0.5633333333333334
+boolq,3,GPT-3 Style,acc,0.5316666666666666
+boolq,3,after_reading,acc,0.5646666666666667
+boolq,3,exercise,acc,0.5626666666666666
+boolq,3,valid_binary,acc,0.5583333333333333
+boolq,3,yes_no_question,acc,0.5976666666666667
+boolq,3,median,accuracy,0.5626666666666666
+boolq,4,GPT-3 Style,acc,0.5383333333333333
+boolq,4,after_reading,acc,0.5673333333333334
+boolq,4,exercise,acc,0.5706666666666667
+boolq,4,valid_binary,acc,0.551
+boolq,4,yes_no_question,acc,0.6023333333333334
+boolq,4,median,accuracy,0.5673333333333334
+boolq,5,GPT-3 Style,acc,0.5336666666666666
+boolq,5,after_reading,acc,0.5746666666666667
+boolq,5,exercise,acc,0.5703333333333334
+boolq,5,valid_binary,acc,0.5626666666666666
+boolq,5,yes_no_question,acc,0.6023333333333334
+boolq,5,median,accuracy,0.5703333333333334
+boolq,5,average,multiple,0.5716666666666667
+cb,0,GPT-3 style,acc,0.19642857142857142
+cb,0,MNLI crowdsource,acc,0.4107142857142857
+cb,0,can we infer,acc,0.5535714285714286
+cb,0,guaranteed/possible/impossible,acc,0.23214285714285715
+cb,0,justified in saying,acc,0.5357142857142857
+cb,0,median,accuracy,0.4107142857142857
+cb,1,GPT-3 style,acc,0.44642857142857145
+cb,1,MNLI crowdsource,acc,0.39285714285714285
+cb,1,can we infer,acc,0.4107142857142857
+cb,1,guaranteed/possible/impossible,acc,0.39285714285714285
+cb,1,justified in saying,acc,0.44642857142857145
+cb,1,median,accuracy,0.4107142857142857
+cb,2,GPT-3 style,acc,0.42857142857142855
+cb,2,MNLI crowdsource,acc,0.39285714285714285
+cb,2,can we infer,acc,0.44642857142857145
+cb,2,guaranteed/possible/impossible,acc,0.39285714285714285
+cb,2,justified in saying,acc,0.44642857142857145
+cb,2,median,accuracy,0.42857142857142855
+cb,3,GPT-3 style,acc,0.4107142857142857
+cb,3,MNLI crowdsource,acc,0.42857142857142855
+cb,3,can we infer,acc,0.4107142857142857
+cb,3,guaranteed/possible/impossible,acc,0.39285714285714285
+cb,3,justified in saying,acc,0.4107142857142857
+cb,3,median,accuracy,0.4107142857142857
+cb,4,GPT-3 style,acc,0.42857142857142855
+cb,4,MNLI crowdsource,acc,0.5
+cb,4,can we infer,acc,0.39285714285714285
+cb,4,guaranteed/possible/impossible,acc,0.39285714285714285
+cb,4,justified in saying,acc,0.42857142857142855
+cb,4,median,accuracy,0.42857142857142855
+cb,5,GPT-3 style,acc,0.39285714285714285
+cb,5,MNLI crowdsource,acc,0.4642857142857143
+cb,5,can we infer,acc,0.42857142857142855
+cb,5,guaranteed/possible/impossible,acc,0.4107142857142857
+cb,5,justified in saying,acc,0.44642857142857145
+cb,5,median,accuracy,0.42857142857142855
+cb,5,average,multiple,0.4196428571428571
+copa,0,best_option,acc,0.6
+copa,0,cause_effect,acc,0.56
+copa,0,choose,acc,0.59
+copa,0,i_am_hesitating,acc,0.59
+copa,0,plausible_alternatives,acc,0.58
+copa,0,median,accuracy,0.59
+copa,1,best_option,acc,0.5
+copa,1,cause_effect,acc,0.57
+copa,1,choose,acc,0.57
+copa,1,i_am_hesitating,acc,0.55
+copa,1,plausible_alternatives,acc,0.56
+copa,1,median,accuracy,0.56
+copa,2,best_option,acc,0.52
+copa,2,cause_effect,acc,0.56
+copa,2,choose,acc,0.58
+copa,2,i_am_hesitating,acc,0.56
+copa,2,plausible_alternatives,acc,0.58
+copa,2,median,accuracy,0.56
+copa,3,best_option,acc,0.61
+copa,3,cause_effect,acc,0.59
+copa,3,choose,acc,0.61
+copa,3,i_am_hesitating,acc,0.57
+copa,3,plausible_alternatives,acc,0.6
+copa,3,median,accuracy,0.6
+copa,4,best_option,acc,0.58
+copa,4,cause_effect,acc,0.56
+copa,4,choose,acc,0.59
+copa,4,i_am_hesitating,acc,0.56
+copa,4,plausible_alternatives,acc,0.55
+copa,4,median,accuracy,0.56
+copa,5,best_option,acc,0.58
+copa,5,cause_effect,acc,0.55
+copa,5,choose,acc,0.57
+copa,5,i_am_hesitating,acc,0.57
+copa,5,plausible_alternatives,acc,0.56
+copa,5,median,accuracy,0.57
+copa,5,average,multiple,0.5733333333333334
+e2e_nlg_cleaned,0,coherent_text,rouge2_fmeasure,0.07683031179945712
+e2e_nlg_cleaned,0,create_text_for_me,rouge2_fmeasure,0.12592500811764892
+e2e_nlg_cleaned,0,generate_gramatically_correct_text,rouge2_fmeasure,0.03866825902601929
+e2e_nlg_cleaned,0,generate_text_restaurant,rouge2_fmeasure,0.02216822438561205
+e2e_nlg_cleaned,0,text,rouge2_fmeasure,0.18707996026648183
+e2e_nlg_cleaned,0,median,rouge2_fmeasure,0.07683031179945712
+e2e_nlg_cleaned,1,coherent_text,rouge2_fmeasure,0.04696802025613863
+e2e_nlg_cleaned,1,create_text_for_me,rouge2_fmeasure,0.05060323801799519
+e2e_nlg_cleaned,1,generate_gramatically_correct_text,rouge2_fmeasure,0.08617713780376496
+e2e_nlg_cleaned,1,generate_text_restaurant,rouge2_fmeasure,0.07230219657399567
+e2e_nlg_cleaned,1,text,rouge2_fmeasure,0.0752688547278534
+e2e_nlg_cleaned,1,median,rouge2_fmeasure,0.07230219657399567
+e2e_nlg_cleaned,2,coherent_text,rouge2_fmeasure,0.07274573138257315
+e2e_nlg_cleaned,2,create_text_for_me,rouge2_fmeasure,0.07551724406985702
+e2e_nlg_cleaned,2,generate_gramatically_correct_text,rouge2_fmeasure,0.08158846564278203
+e2e_nlg_cleaned,2,generate_text_restaurant,rouge2_fmeasure,0.07905711793730946
+e2e_nlg_cleaned,2,text,rouge2_fmeasure,0.0831703337253987
+e2e_nlg_cleaned,2,median,rouge2_fmeasure,0.07905711793730946
+e2e_nlg_cleaned,3,coherent_text,rouge2_fmeasure,0.08511598673947676
+e2e_nlg_cleaned,3,create_text_for_me,rouge2_fmeasure,0.08225064700727518
+e2e_nlg_cleaned,3,generate_gramatically_correct_text,rouge2_fmeasure,0.07939963217690697
+e2e_nlg_cleaned,3,generate_text_restaurant,rouge2_fmeasure,0.08864387470534706
+e2e_nlg_cleaned,3,text,rouge2_fmeasure,0.09057857385566902
+e2e_nlg_cleaned,3,median,rouge2_fmeasure,0.08511598673947676
+e2e_nlg_cleaned,4,coherent_text,rouge2_fmeasure,0.0903654887337426
+e2e_nlg_cleaned,4,create_text_for_me,rouge2_fmeasure,0.08832498480105934
+e2e_nlg_cleaned,4,generate_gramatically_correct_text,rouge2_fmeasure,0.07785610993313638
+e2e_nlg_cleaned,4,generate_text_restaurant,rouge2_fmeasure,0.09381098739864982
+e2e_nlg_cleaned,4,text,rouge2_fmeasure,0.0944527466973476
+e2e_nlg_cleaned,4,median,rouge2_fmeasure,0.0903654887337426
+e2e_nlg_cleaned,5,coherent_text,rouge2_fmeasure,0.09565357595328416
+e2e_nlg_cleaned,5,create_text_for_me,rouge2_fmeasure,0.09313015798234085
+e2e_nlg_cleaned,5,generate_gramatically_correct_text,rouge2_fmeasure,0.07759858672584617
+e2e_nlg_cleaned,5,generate_text_restaurant,rouge2_fmeasure,0.09730358308616667
+e2e_nlg_cleaned,5,text,rouge2_fmeasure,0.0981873842191505
+e2e_nlg_cleaned,5,median,rouge2_fmeasure,0.09565357595328416
+e2e_nlg_cleaned,5,average,multiple,0.08322077962287763
+gem_xsum,0,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.014331752023111665
+gem_xsum,0,DOC_tldr,rouge2_fmeasure,0.015793527267809353
+gem_xsum,0,article_DOC_summary,rouge2_fmeasure,0.01783873426577586
+gem_xsum,0,summarize_DOC,rouge2_fmeasure,0.01634264454041957
+gem_xsum,0,summarize_this_DOC_summary,rouge2_fmeasure,0.016637993147190867
+gem_xsum,0,median,rouge2_fmeasure,0.01634264454041957
+gem_xsum,1,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.01563559121007456
+gem_xsum,1,DOC_tldr,rouge2_fmeasure,0.012260258344477375
+gem_xsum,1,article_DOC_summary,rouge2_fmeasure,0.02086687296490721
+gem_xsum,1,summarize_DOC,rouge2_fmeasure,0.017118068362416517
+gem_xsum,1,summarize_this_DOC_summary,rouge2_fmeasure,0.01660477741264796
+gem_xsum,1,median,rouge2_fmeasure,0.01660477741264796
+gem_xsum,2,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.01551951461927907
+gem_xsum,2,DOC_tldr,rouge2_fmeasure,0.012201390939302973
+gem_xsum,2,article_DOC_summary,rouge2_fmeasure,0.015790474040565182
+gem_xsum,2,summarize_DOC,rouge2_fmeasure,0.015337369323206902
+gem_xsum,2,summarize_this_DOC_summary,rouge2_fmeasure,0.015360819588816904
+gem_xsum,2,median,rouge2_fmeasure,0.015360819588816904
+gem_xsum,3,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.014615300016460326
+gem_xsum,3,DOC_tldr,rouge2_fmeasure,0.014849672103797189
+gem_xsum,3,article_DOC_summary,rouge2_fmeasure,0.015065187296172565
+gem_xsum,3,summarize_DOC,rouge2_fmeasure,0.014667451968252554
+gem_xsum,3,summarize_this_DOC_summary,rouge2_fmeasure,0.014649332777625616
+gem_xsum,3,median,rouge2_fmeasure,0.014667451968252554
+gem_xsum,4,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.004331423800367673
+gem_xsum,4,DOC_tldr,rouge2_fmeasure,0.004600261742688949
+gem_xsum,4,article_DOC_summary,rouge2_fmeasure,0.004530040031767069
+gem_xsum,4,summarize_DOC,rouge2_fmeasure,0.004765891573358166
+gem_xsum,4,summarize_this_DOC_summary,rouge2_fmeasure,0.004119906223301282
+gem_xsum,4,median,rouge2_fmeasure,0.004530040031767069
+gem_xsum,5,DOC_boils_down_to_simple_idea_that,rouge2_fmeasure,0.0
+gem_xsum,5,DOC_tldr,rouge2_fmeasure,0.0001270035372229107
+gem_xsum,5,article_DOC_summary,rouge2_fmeasure,0.00021112542779058987
+gem_xsum,5,summarize_DOC,rouge2_fmeasure,0.000538565784940234
+gem_xsum,5,summarize_this_DOC_summary,rouge2_fmeasure,0.00013885485583598793
+gem_xsum,5,median,rouge2_fmeasure,0.00013885485583598793
+gem_xsum,5,average,multiple,0.011274098066290008
+piqa,0,Correct the solution,rouge2_fmeasure,0.06256105007088962
+piqa,0,choose the most appropriate solution,acc,0.5048966267682263
+piqa,0,no prompt needed,rouge2_fmeasure,0.0024039653739287383
+piqa,0,pick_correct_choice_index,acc,0.49510337323177367
+piqa,0,what_is_the_correct_ending,acc,0.4956474428726877
+piqa,0,median,accuracy,0.4956474428726877
+piqa,1,Correct the solution,rouge2_fmeasure,0.012117898396108417
+piqa,1,choose the most appropriate solution,acc,0.5016322089227421
+piqa,1,no prompt needed,rouge2_fmeasure,0.005941822792449927
+piqa,1,pick_correct_choice_index,acc,0.499455930359086
+piqa,1,what_is_the_correct_ending,acc,0.5092491838955386
+piqa,1,median,accuracy,0.5016322089227421
+piqa,2,Correct the solution,rouge2_fmeasure,0.023433042203883557
+piqa,2,choose the most appropriate solution,acc,0.5048966267682263
+piqa,2,no prompt needed,rouge2_fmeasure,0.005293218483445248
+piqa,2,pick_correct_choice_index,acc,0.5054406964091404
+piqa,2,what_is_the_correct_ending,acc,0.5108813928182807
+piqa,2,median,accuracy,0.5054406964091404
+piqa,3,Correct the solution,rouge2_fmeasure,0.029903412643591754
+piqa,3,choose the most appropriate solution,acc,0.5272034820457019
+piqa,3,no prompt needed,rouge2_fmeasure,0.005307127222312835
+piqa,3,pick_correct_choice_index,acc,0.515233949945593
+piqa,3,what_is_the_correct_ending,acc,0.515233949945593
+piqa,3,median,accuracy,0.515233949945593
+piqa,4,Correct the solution,rouge2_fmeasure,0.03508516074414074
+piqa,4,choose the most appropriate solution,acc,0.5125136017410229
+piqa,4,no prompt needed,rouge2_fmeasure,0.005146040515208341
+piqa,4,pick_correct_choice_index,acc,0.514145810663765
+piqa,4,what_is_the_correct_ending,acc,0.5114254624591947
+piqa,4,median,accuracy,0.5125136017410229
+piqa,5,Correct the solution,rouge2_fmeasure,0.03697008601032397
+piqa,5,choose the most appropriate solution,acc,0.5130576713819369
+piqa,5,no prompt needed,rouge2_fmeasure,0.005234740517822527
+piqa,5,pick_correct_choice_index,acc,0.5087051142546246
+piqa,5,what_is_the_correct_ending,acc,0.5048966267682263
+piqa,5,median,accuracy,0.5087051142546246
+piqa,5,average,multiple,0.5065288356909684
+sciq,0,Direct Question,acc,0.561
+sciq,0,Direct Question (Closed Book),acc,0.296
+sciq,0,Multiple Choice,acc,0.326
+sciq,0,Multiple Choice (Closed Book),acc,0.302
+sciq,0,Multiple Choice Question First,acc,0.353
+sciq,0,median,accuracy,0.326
+sciq,1,Direct Question,acc,0.578
+sciq,1,Direct Question (Closed Book),acc,0.3
+sciq,1,Multiple Choice,acc,0.329
+sciq,1,Multiple Choice (Closed Book),acc,0.32
+sciq,1,Multiple Choice Question First,acc,0.362
+sciq,1,median,accuracy,0.329
+sciq,2,Direct Question,acc,0.563
+sciq,2,Direct Question (Closed Book),acc,0.299
+sciq,2,Multiple Choice,acc,0.344
+sciq,2,Multiple Choice (Closed Book),acc,0.33
+sciq,2,Multiple Choice Question First,acc,0.374
+sciq,2,median,accuracy,0.344
+sciq,3,Direct Question,acc,0.567
+sciq,3,Direct Question (Closed Book),acc,0.301
+sciq,3,Multiple Choice,acc,0.362
+sciq,3,Multiple Choice (Closed Book),acc,0.338
+sciq,3,Multiple Choice Question First,acc,0.38
+sciq,3,median,accuracy,0.362
+sciq,4,Direct Question,acc,0.573
+sciq,4,Direct Question (Closed Book),acc,0.309
+sciq,4,Multiple Choice,acc,0.356
+sciq,4,Multiple Choice (Closed Book),acc,0.345
+sciq,4,Multiple Choice Question First,acc,0.37
+sciq,4,median,accuracy,0.356
+sciq,5,Direct Question,acc,0.573
+sciq,5,Direct Question (Closed Book),acc,0.303
+sciq,5,Multiple Choice,acc,0.344
+sciq,5,Multiple Choice (Closed Book),acc,0.325
+sciq,5,Multiple Choice Question First,acc,0.375
+sciq,5,median,accuracy,0.344
+sciq,5,average,multiple,0.34349999999999997
+story_cloze_2016,0,Answer Given options,acc,0.5056119722073757
+story_cloze_2016,0,Choose Story Ending,acc,0.518439337252806
+story_cloze_2016,0,Novel Correct Ending,acc,0.5008017103153394
+story_cloze_2016,0,Story Continuation and Options,acc,0.5168359166221272
+story_cloze_2016,0,median,accuracy,0.5112239444147515
+story_cloze_2016,1,Answer Given options,acc,0.49812934259754144
+story_cloze_2016,1,Choose Story Ending,acc,0.5168359166221272
+story_cloze_2016,1,Novel Correct Ending,acc,0.4863709246392304
+story_cloze_2016,1,Story Continuation and Options,acc,0.5120256547300909
+story_cloze_2016,1,median,accuracy,0.5050774986638161
+story_cloze_2016,2,Answer Given options,acc,0.4949225013361839
+story_cloze_2016,2,Choose Story Ending,acc,0.51309460181721
+story_cloze_2016,2,Novel Correct Ending,acc,0.48583645109567075
+story_cloze_2016,2,Story Continuation and Options,acc,0.5066809192944949
+story_cloze_2016,2,median,accuracy,0.5008017103153394
+story_cloze_2016,3,Answer Given options,acc,0.48850881881346875
+story_cloze_2016,3,Choose Story Ending,acc,0.5056119722073757
+story_cloze_2016,3,Novel Correct Ending,acc,0.4836985569214324
+story_cloze_2016,3,Story Continuation and Options,acc,0.4922501336183859
+story_cloze_2016,3,median,accuracy,0.4903794762159273
+story_cloze_2016,4,Answer Given options,acc,0.4965259219668626
+story_cloze_2016,4,Choose Story Ending,acc,0.5125601282736505
+story_cloze_2016,4,Novel Correct Ending,acc,0.4826296098343132
+story_cloze_2016,4,Story Continuation and Options,acc,0.5018706574024586
+story_cloze_2016,4,median,accuracy,0.4991982896846606
+story_cloze_2016,5,Answer Given options,acc,0.49331908070550506
+story_cloze_2016,5,Choose Story Ending,acc,0.5056119722073757
+story_cloze_2016,5,Novel Correct Ending,acc,0.4890432923570283
+story_cloze_2016,5,Story Continuation and Options,acc,0.498663816141101
+story_cloze_2016,5,median,accuracy,0.495991448423303
+story_cloze_2016,5,average,multiple,0.500445394619633
+superglue_rte,0,GPT-3 style,acc,0.5270758122743683
+superglue_rte,0,MNLI crowdsource,acc,0.516245487364621
+superglue_rte,0,does it follow that,acc,0.4368231046931408
+superglue_rte,0,guaranteed true,acc,0.49458483754512633
+superglue_rte,0,should assume,acc,0.5270758122743683
+superglue_rte,0,median,accuracy,0.516245487364621
+superglue_rte,1,GPT-3 style,acc,0.49458483754512633
+superglue_rte,1,MNLI crowdsource,acc,0.49097472924187724
+superglue_rte,1,does it follow that,acc,0.49097472924187724
+superglue_rte,1,guaranteed true,acc,0.49097472924187724
+superglue_rte,1,should assume,acc,0.49097472924187724
+superglue_rte,1,median,accuracy,0.49097472924187724
+superglue_rte,2,GPT-3 style,acc,0.49458483754512633
+superglue_rte,2,MNLI crowdsource,acc,0.4981949458483754
+superglue_rte,2,does it follow that,acc,0.48736462093862815
+superglue_rte,2,guaranteed true,acc,0.47653429602888087
+superglue_rte,2,should assume,acc,0.48014440433212996
+superglue_rte,2,median,accuracy,0.48736462093862815
+superglue_rte,3,GPT-3 style,acc,0.49458483754512633
+superglue_rte,3,MNLI crowdsource,acc,0.4981949458483754
+superglue_rte,3,does it follow that,acc,0.47653429602888087
+superglue_rte,3,guaranteed true,acc,0.49458483754512633
+superglue_rte,3,should assume,acc,0.48375451263537905
+superglue_rte,3,median,accuracy,0.49458483754512633
+superglue_rte,4,GPT-3 style,acc,0.48014440433212996
+superglue_rte,4,MNLI crowdsource,acc,0.5054151624548736
+superglue_rte,4,does it follow that,acc,0.4729241877256318
+superglue_rte,4,guaranteed true,acc,0.49097472924187724
+superglue_rte,4,should assume,acc,0.48014440433212996
+superglue_rte,4,median,accuracy,0.48014440433212996
+superglue_rte,5,GPT-3 style,acc,0.48736462093862815
+superglue_rte,5,MNLI crowdsource,acc,0.48375451263537905
+superglue_rte,5,does it follow that,acc,0.4620938628158845
+superglue_rte,5,guaranteed true,acc,0.4981949458483754
+superglue_rte,5,should assume,acc,0.4981949458483754
+superglue_rte,5,median,accuracy,0.48736462093862815
+superglue_rte,5,average,multiple,0.4927797833935018
+web_nlg_en,0,PALM_prompt,rouge2_fmeasure,0.031734228703201264
+web_nlg_en,0,explicit-graph-description2,rouge2_fmeasure,0.0021022925642461623
+web_nlg_en,0,implicit-graph-description,rouge2_fmeasure,0.004097730822753029
+web_nlg_en,0,non-explicit-description,rouge2_fmeasure,0.004339378512582052
+web_nlg_en,0,very-explicit-description,rouge2_fmeasure,0.04755242841283524
+web_nlg_en,0,median,rouge2_fmeasure,0.004339378512582052
+web_nlg_en,1,PALM_prompt,rouge2_fmeasure,0.029924776302610284
+web_nlg_en,1,explicit-graph-description2,rouge2_fmeasure,0.013624757551384124
+web_nlg_en,1,implicit-graph-description,rouge2_fmeasure,0.013747268988959595
+web_nlg_en,1,non-explicit-description,rouge2_fmeasure,0.013642298532841417
+web_nlg_en,1,very-explicit-description,rouge2_fmeasure,0.014903773827883306
+web_nlg_en,1,median,rouge2_fmeasure,0.013747268988959595
+web_nlg_en,2,PALM_prompt,rouge2_fmeasure,0.030198905160389044
+web_nlg_en,2,explicit-graph-description2,rouge2_fmeasure,0.014100180203474038
+web_nlg_en,2,implicit-graph-description,rouge2_fmeasure,0.013404518942651407
+web_nlg_en,2,non-explicit-description,rouge2_fmeasure,0.013089719060620607
+web_nlg_en,2,very-explicit-description,rouge2_fmeasure,0.014494882572376312
+web_nlg_en,2,median,rouge2_fmeasure,0.014100180203474038
+web_nlg_en,3,PALM_prompt,rouge2_fmeasure,0.0299181905955771
+web_nlg_en,3,explicit-graph-description2,rouge2_fmeasure,0.013249922136095603
+web_nlg_en,3,implicit-graph-description,rouge2_fmeasure,0.013694547280363368
+web_nlg_en,3,non-explicit-description,rouge2_fmeasure,0.013153025539986579
+web_nlg_en,3,very-explicit-description,rouge2_fmeasure,0.013187050625056523
+web_nlg_en,3,median,rouge2_fmeasure,0.013249922136095603
+web_nlg_en,4,PALM_prompt,rouge2_fmeasure,0.031639571635299846
+web_nlg_en,4,explicit-graph-description2,rouge2_fmeasure,0.014229702953664455
+web_nlg_en,4,implicit-graph-description,rouge2_fmeasure,0.01467598496147871
+web_nlg_en,4,non-explicit-description,rouge2_fmeasure,0.013872966659414053
+web_nlg_en,4,very-explicit-description,rouge2_fmeasure,0.014802883130151939
+web_nlg_en,4,median,rouge2_fmeasure,0.01467598496147871
+web_nlg_en,5,PALM_prompt,rouge2_fmeasure,0.030022553056029706
+web_nlg_en,5,explicit-graph-description2,rouge2_fmeasure,0.015163133405164434
+web_nlg_en,5,implicit-graph-description,rouge2_fmeasure,0.01573629832661093
+web_nlg_en,5,non-explicit-description,rouge2_fmeasure,0.013595010492421218
+web_nlg_en,5,very-explicit-description,rouge2_fmeasure,0.01624180543052662
+web_nlg_en,5,median,rouge2_fmeasure,0.01573629832661093
+web_nlg_en,5,average,multiple,0.012641505521533487
+wiki_lingua_en,0,article_summary_en,rouge2_fmeasure,0.052567975188848405
+wiki_lingua_en,0,rephrase_en,rouge2_fmeasure,0.007854289251701565
+wiki_lingua_en,0,summarize_above_en,rouge2_fmeasure,0.0038458659144674466
+wiki_lingua_en,0,tldr_en,rouge2_fmeasure,0.015671748077472996
+wiki_lingua_en,0,write_abstract_en,rouge2_fmeasure,0.014303497306763413
+wiki_lingua_en,0,median,rouge2_fmeasure,0.014303497306763413
+wiki_lingua_en,1,article_summary_en,rouge2_fmeasure,0.03761257717991928
+wiki_lingua_en,1,rephrase_en,rouge2_fmeasure,0.004986359493861402
+wiki_lingua_en,1,summarize_above_en,rouge2_fmeasure,0.0038867114952000873
+wiki_lingua_en,1,tldr_en,rouge2_fmeasure,0.005755944338368179
+wiki_lingua_en,1,write_abstract_en,rouge2_fmeasure,0.0052645090649109895
+wiki_lingua_en,1,median,rouge2_fmeasure,0.0052645090649109895
+wiki_lingua_en,2,article_summary_en,rouge2_fmeasure,0.025570973012891195
+wiki_lingua_en,2,rephrase_en,rouge2_fmeasure,0.00600392083872442
+wiki_lingua_en,2,summarize_above_en,rouge2_fmeasure,0.005194947294074261
+wiki_lingua_en,2,tldr_en,rouge2_fmeasure,0.010504603460980755
+wiki_lingua_en,2,write_abstract_en,rouge2_fmeasure,0.006048464517830181
+wiki_lingua_en,2,median,rouge2_fmeasure,0.006048464517830181
+wiki_lingua_en,3,article_summary_en,rouge2_fmeasure,0.02030504236701163
+wiki_lingua_en,3,rephrase_en,rouge2_fmeasure,0.006494830300354191
+wiki_lingua_en,3,summarize_above_en,rouge2_fmeasure,0.00471973086085565
+wiki_lingua_en,3,tldr_en,rouge2_fmeasure,0.01096682747964428
+wiki_lingua_en,3,write_abstract_en,rouge2_fmeasure,0.0058753493744521336
+wiki_lingua_en,3,median,rouge2_fmeasure,0.006494830300354191
+wiki_lingua_en,4,article_summary_en,rouge2_fmeasure,0.006731993501111477
+wiki_lingua_en,4,rephrase_en,rouge2_fmeasure,0.002417994702779381
+wiki_lingua_en,4,summarize_above_en,rouge2_fmeasure,0.0018715335775846497
+wiki_lingua_en,4,tldr_en,rouge2_fmeasure,0.005041922188233573
+wiki_lingua_en,4,write_abstract_en,rouge2_fmeasure,0.0018364476816945278
+wiki_lingua_en,4,median,rouge2_fmeasure,0.002417994702779381
+wiki_lingua_en,5,article_summary_en,rouge2_fmeasure,0.0012665122562968891
+wiki_lingua_en,5,rephrase_en,rouge2_fmeasure,0.00036529417385801794
+wiki_lingua_en,5,summarize_above_en,rouge2_fmeasure,0.00020048952920264672
+wiki_lingua_en,5,tldr_en,rouge2_fmeasure,0.0010361138600270753
+wiki_lingua_en,5,write_abstract_en,rouge2_fmeasure,0.0002340501971901335
+wiki_lingua_en,5,median,rouge2_fmeasure,0.00036529417385801794
+wiki_lingua_en,5,average,multiple,0.005815765011082695
+winogrande,0,Replace,acc,0.5059194948697711
+winogrande,0,True or False,acc,0.4980268350434096
+winogrande,0,does underscore refer to,acc,0.4877663772691397
+winogrande,0,stand for,acc,0.4925019731649566
+winogrande,0,underscore refer to,acc,0.48855564325177586
+winogrande,0,median,accuracy,0.4925019731649566
+winogrande,1,Replace,acc,0.49329123914759276
+winogrande,1,True or False,acc,0.4940805051302289
+winogrande,1,does underscore refer to,acc,0.4877663772691397
+winogrande,1,stand for,acc,0.500394632991318
+winogrande,1,underscore refer to,acc,0.48539857932123126
+winogrande,1,median,accuracy,0.49329123914759276
+winogrande,2,Replace,acc,0.4877663772691397
+winogrande,2,True or False,acc,0.5035516969218626
+winogrande,2,does underscore refer to,acc,0.5201262825572218
+winogrande,2,stand for,acc,0.5169692186266772
+winogrande,2,underscore refer to,acc,0.4925019731649566
+winogrande,2,median,accuracy,0.5035516969218626
+winogrande,3,Replace,acc,0.505130228887135
+winogrande,3,True or False,acc,0.5059194948697711
+winogrande,3,does underscore refer to,acc,0.5122336227308603
+winogrande,3,stand for,acc,0.5122336227308603
+winogrande,3,underscore refer to,acc,0.5074980268350434
+winogrande,3,median,accuracy,0.5074980268350434
+winogrande,4,Replace,acc,0.510655090765588
+winogrande,4,True or False,acc,0.526440410418311
+winogrande,4,does underscore refer to,acc,0.5114443567482242
+winogrande,4,stand for,acc,0.5098658247829518
+winogrande,4,underscore refer to,acc,0.500394632991318
+winogrande,4,median,accuracy,0.510655090765588
+winogrande,5,Replace,acc,0.505130228887135
+winogrande,5,True or False,acc,0.5122336227308603
+winogrande,5,does underscore refer to,acc,0.489344909234412
+winogrande,5,stand for,acc,0.5090765588003157
+winogrande,5,underscore refer to,acc,0.4988161010260458
+winogrande,5,median,accuracy,0.505130228887135
+winogrande,5,average,multiple,0.5021047092870298
diff --git a/1b11b51b5/eval/merged.json b/1b11b51b5/eval/merged.json
new file mode 100644
index 0000000000000000000000000000000000000000..632f874af18481e6176dba53a8aa9a50c41a2816
--- /dev/null
+++ b/1b11b51b5/eval/merged.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"0": {"PALM_prompt": {"bleu": 0.1295929648118789, "bleu_stderr": 0.017717451610532654, "rouge1_fmeasure": 0.07130151635044231, "rouge1_fmeasure_stderr": 0.0017145172534413066, "rouge1_precision": 0.057688485079104755, "rouge1_precision_stderr": 0.002659417350231105, "rouge1_recall": 0.18771361409144396, "rouge1_recall_stderr": 0.004194847117781067, "rouge2_fmeasure": 0.031734228703201264, "rouge2_fmeasure_stderr": 0.0010224014054478407, "rouge2_precision": 0.022591789424650413, "rouge2_precision_stderr": 0.0012409453957858067, "rouge2_recall": 0.08711168383171403, "rouge2_recall_stderr": 0.0026849545852757613, "rougeL_fmeasure": 0.0701415239947909, "rougeL_fmeasure_stderr": 0.0016880620041233863, "rougeL_precision": 0.05685991375829496, "rougeL_precision_stderr": 0.0026473311931234454, "rougeL_recall": 0.18500805358362152, "rougeL_recall_stderr": 0.004147431246244616, "rougeLsum_fmeasure": 0.068682573498121, "rougeLsum_fmeasure_stderr": 0.001637051783542573, "rougeLsum_precision": 0.05595104569982353, "rougeLsum_precision_stderr": 0.002636691350690872, "rougeLsum_recall": 0.1806575902205997, "rougeLsum_recall_stderr": 0.003940723272696618}, "explicit-graph-description2": {"bleu": 0.007162912027319248, "bleu_stderr": 8.799732134630633e-05, "rouge1_fmeasure": 0.04370443194320921, "rouge1_fmeasure_stderr": 0.0008270949225797383, "rouge1_precision": 0.02716794991360227, "rouge1_precision_stderr": 0.0008204423792394508, "rouge1_recall": 0.1822527382056869, "rouge1_recall_stderr": 0.002246189358522882, "rouge2_fmeasure": 0.0021022925642461623, "rouge2_fmeasure_stderr": 0.00011341026500721504, "rouge2_precision": 0.001333594787639512, "rouge2_precision_stderr": 0.00011638013641618982, "rouge2_recall": 0.009285831703201557, "rouge2_recall_stderr": 0.0005095411845886788, "rougeL_fmeasure": 0.043375077921510685, "rougeL_fmeasure_stderr": 0.0008125955785869596, "rougeL_precision": 0.026796470589552018, "rougeL_precision_stderr": 0.0007542828958134997, "rougeL_recall": 0.1812832654319008, "rougeL_recall_stderr": 0.002242575337885422, "rougeLsum_fmeasure": 0.031579687187122944, "rougeLsum_fmeasure_stderr": 0.000632968954798297, "rougeLsum_precision": 0.019859301830203156, "rougeLsum_precision_stderr": 0.000705291344078074, "rougeLsum_recall": 0.13504875184894552, "rougeLsum_recall_stderr": 0.0017157651520892864}, "implicit-graph-description": {"bleu": 0.04787169131286324, "bleu_stderr": 0.01817594262932824, "rouge1_fmeasure": 0.04686425613621658, "rouge1_fmeasure_stderr": 0.0008219392538504159, "rouge1_precision": 0.05219573618812013, "rouge1_precision_stderr": 0.0036463591354812734, "rouge1_recall": 0.2092941055787021, "rouge1_recall_stderr": 0.002738958078725276, "rouge2_fmeasure": 0.004097730822753029, "rouge2_fmeasure_stderr": 0.000284378780561254, "rouge2_precision": 0.0023893079304512085, "rouge2_precision_stderr": 0.00017566116680500324, "rouge2_recall": 0.020431268269765074, "rouge2_recall_stderr": 0.0011345589198703068, "rougeL_fmeasure": 0.04594073467891509, "rougeL_fmeasure_stderr": 0.0007486823064547486, "rougeL_precision": 0.05149194714448206, "rougeL_precision_stderr": 0.0036328394668563605, "rougeL_recall": 0.2070204298599423, "rougeL_recall_stderr": 0.0026942905482556577, "rougeLsum_fmeasure": 0.03389173137151838, "rougeLsum_fmeasure_stderr": 0.0006685380625775191, "rougeLsum_precision": 0.044498915692550794, "rougeLsum_precision_stderr": 0.0036541031800773204, "rougeLsum_recall": 0.15356732693991013, "rougeLsum_recall_stderr": 0.0021296241897594786}, "non-explicit-description": {"bleu": 0.011683013048448883, "bleu_stderr": 0.002299898655396354, "rouge1_fmeasure": 0.03521658777407273, "rouge1_fmeasure_stderr": 0.001032855438532496, "rouge1_precision": 0.024257094385346158, "rouge1_precision_stderr": 0.0012542520013724245, "rouge1_recall": 0.14438146327799528, "rouge1_recall_stderr": 0.002210767634374098, "rouge2_fmeasure": 0.004339378512582052, "rouge2_fmeasure_stderr": 0.0006544799388018146, "rouge2_precision": 0.0036895014079609267, "rouge2_precision_stderr": 0.0006419014802940976, "rouge2_recall": 0.01251292918426236, "rouge2_recall_stderr": 0.0012328752315072008, "rougeL_fmeasure": 0.03281178600545513, "rougeL_fmeasure_stderr": 0.0008979493883162579, "rougeL_precision": 0.022267351364883243, "rougeL_precision_stderr": 0.001060310841195819, "rougeL_recall": 0.137175853341158, "rougeL_recall_stderr": 0.0021144498099671533, "rougeLsum_fmeasure": 0.03131853952441433, "rougeLsum_fmeasure_stderr": 0.000904933217930217, "rougeLsum_precision": 0.021691614849279027, "rougeLsum_precision_stderr": 0.0011300013638413863, "rougeLsum_recall": 0.1294165016082978, "rougeLsum_recall_stderr": 0.0019946881884391385}, "very-explicit-description": {"bleu": 0.4105317095314889, "bleu_stderr": 0.043066710267683765, "rouge1_fmeasure": 0.1338747913042839, "rouge1_fmeasure_stderr": 0.002030240874957092, "rouge1_precision": 0.08308142158928723, "rouge1_precision_stderr": 0.0014822533733938113, "rouge1_recall": 0.4813530094293165, "rouge1_recall_stderr": 0.0052207165316912555, "rouge2_fmeasure": 0.04755242841283524, "rouge2_fmeasure_stderr": 0.0011842958136442528, "rouge2_precision": 0.029535536265787973, "rouge2_precision_stderr": 0.0008372816447783799, "rouge2_recall": 0.17908532550201076, "rouge2_recall_stderr": 0.003415407692561151, "rougeL_fmeasure": 0.1175214238809207, "rougeL_fmeasure_stderr": 0.0015866825442232382, "rougeL_precision": 0.07204203263806305, "rougeL_precision_stderr": 0.0011304033839566587, "rougeL_recall": 0.44548440426035085, "rougeL_recall_stderr": 0.00514873304330449, "rougeLsum_fmeasure": 0.11176670755192196, "rougeLsum_fmeasure_stderr": 0.001789832056695336, "rougeLsum_precision": 0.06951673475733636, "rougeLsum_precision_stderr": 0.0013120942001119277, "rougeLsum_recall": 0.40195435934423007, "rougeLsum_recall_stderr": 0.004355224640972606}}, "1": {"PALM_prompt": {"bleu": 0.14987013983247244, "bleu_stderr": 0.01941974811684623, "rouge1_fmeasure": 0.07380028517409874, "rouge1_fmeasure_stderr": 0.0017644455999507468, "rouge1_precision": 0.05507727706640666, "rouge1_precision_stderr": 0.0019564677497519853, "rouge1_recall": 0.18027308616485552, "rouge1_recall_stderr": 0.0038002016718867975, "rouge2_fmeasure": 0.029924776302610284, "rouge2_fmeasure_stderr": 0.0010166921536067964, "rouge2_precision": 0.020536550124082592, "rouge2_precision_stderr": 0.0009198527068768572, "rouge2_recall": 0.07773785714938754, "rouge2_recall_stderr": 0.0024576072009639223, "rougeL_fmeasure": 0.07089555573728011, "rougeL_fmeasure_stderr": 0.0016677661366070384, "rougeL_precision": 0.05266527434294661, "rougeL_precision_stderr": 0.0018768852497859027, "rougeL_recall": 0.1755608787259102, "rougeL_recall_stderr": 0.0037271693076169517, "rougeLsum_fmeasure": 0.07071172213779651, "rougeLsum_fmeasure_stderr": 0.001659838827248763, "rougeLsum_precision": 0.05264662810449602, "rougeLsum_precision_stderr": 0.001884149706620751, "rougeLsum_recall": 0.17473281354464437, "rougeLsum_recall_stderr": 0.0036722904268852133}, "explicit-graph-description2": {"bleu": 0.6590131747530226, "bleu_stderr": 0.12977618045440018, "rouge1_fmeasure": 0.14164576016651942, "rouge1_fmeasure_stderr": 0.001778754222107073, "rouge1_precision": 0.16883673090259363, "rouge1_precision_stderr": 0.002502030010371229, "rouge1_recall": 0.15618300732830487, "rouge1_recall_stderr": 0.002359067976848311, "rouge2_fmeasure": 0.013624757551384124, "rouge2_fmeasure_stderr": 0.0007818220844644996, "rouge2_precision": 0.01672633503624124, "rouge2_precision_stderr": 0.0011081302796613816, "rouge2_recall": 0.01600514703432178, "rouge2_recall_stderr": 0.0009715621757517813, "rougeL_fmeasure": 0.11830599709144449, "rougeL_fmeasure_stderr": 0.0014079740228845011, "rougeL_precision": 0.14224381757445975, "rougeL_precision_stderr": 0.00213034490021511, "rougeL_recall": 0.13192814705914896, "rougeL_recall_stderr": 0.00200227535288082, "rougeLsum_fmeasure": 0.1252632960833956, "rougeLsum_fmeasure_stderr": 0.0015337180547933837, "rougeLsum_precision": 0.15042428668427457, "rougeLsum_precision_stderr": 0.002260006746662012, "rougeLsum_recall": 0.13825539582157836, "rougeLsum_recall_stderr": 0.002041174554406684}, "implicit-graph-description": {"bleu": 0.6245795597079693, "bleu_stderr": 0.12364633491228348, "rouge1_fmeasure": 0.1408351138429239, "rouge1_fmeasure_stderr": 0.0017939649759145585, "rouge1_precision": 0.16788566040200872, "rouge1_precision_stderr": 0.002506066712601704, "rouge1_recall": 0.156107822389262, "rouge1_recall_stderr": 0.0023580502662707355, "rouge2_fmeasure": 0.013747268988959595, "rouge2_fmeasure_stderr": 0.0007853529846111664, "rouge2_precision": 0.016831623170465607, "rouge2_precision_stderr": 0.001109690767212444, "rouge2_recall": 0.016312218393737095, "rouge2_recall_stderr": 0.0009932711354024328, "rougeL_fmeasure": 0.11764346217525097, "rougeL_fmeasure_stderr": 0.0014184921221509462, "rougeL_precision": 0.14146179896174868, "rougeL_precision_stderr": 0.00213302407544127, "rougeL_recall": 0.1319410650215301, "rougeL_recall_stderr": 0.001993005317327167, "rougeLsum_fmeasure": 0.12449737336986157, "rougeLsum_fmeasure_stderr": 0.0015465935152696479, "rougeLsum_precision": 0.14956397844019953, "rougeLsum_precision_stderr": 0.0022662888334540306, "rougeLsum_recall": 0.13813269733237477, "rougeLsum_recall_stderr": 0.002042887172507199}, "non-explicit-description": {"bleu": 0.6162564327220701, "bleu_stderr": 0.1172533643555817, "rouge1_fmeasure": 0.14008242251401395, "rouge1_fmeasure_stderr": 0.0017910410554211656, "rouge1_precision": 0.166421902281267, "rouge1_precision_stderr": 0.002509536624513244, "rouge1_recall": 0.15640224145913584, "rouge1_recall_stderr": 0.0023886985315635617, "rouge2_fmeasure": 0.013642298532841417, "rouge2_fmeasure_stderr": 0.00077960580664087, "rouge2_precision": 0.016574599293167843, "rouge2_precision_stderr": 0.001097074412480475, "rouge2_recall": 0.016727604524752332, "rouge2_recall_stderr": 0.0010405236023368633, "rougeL_fmeasure": 0.11709634674078381, "rougeL_fmeasure_stderr": 0.0014146593664078942, "rougeL_precision": 0.14022832153745743, "rougeL_precision_stderr": 0.00213486067462865, "rougeL_recall": 0.13246983853698344, "rougeL_recall_stderr": 0.002033559600741525, "rougeLsum_fmeasure": 0.12378978270420449, "rougeLsum_fmeasure_stderr": 0.0015474580156464902, "rougeLsum_precision": 0.14818197139020592, "rougeLsum_precision_stderr": 0.002271024873481679, "rougeLsum_recall": 0.13825385118721117, "rougeLsum_recall_stderr": 0.002074894761466916}, "very-explicit-description": {"bleu": 0.6314175464516708, "bleu_stderr": 0.12498388493367703, "rouge1_fmeasure": 0.14107370791796536, "rouge1_fmeasure_stderr": 0.0017982597280216904, "rouge1_precision": 0.1662782777493377, "rouge1_precision_stderr": 0.0025085144836969712, "rouge1_recall": 0.16758435156210738, "rouge1_recall_stderr": 0.0030206942122441363, "rouge2_fmeasure": 0.014903773827883306, "rouge2_fmeasure_stderr": 0.0008144412261826056, "rouge2_precision": 0.017259872587569457, "rouge2_precision_stderr": 0.0011062856087889594, "rouge2_recall": 0.022404809879098816, "rouge2_recall_stderr": 0.001426195880760084, "rougeL_fmeasure": 0.11763972203198278, "rougeL_fmeasure_stderr": 0.0014171066452616822, "rougeL_precision": 0.1398613211175107, "rougeL_precision_stderr": 0.002132190748276612, "rougeL_recall": 0.14190066206999388, "rougeL_recall_stderr": 0.0026252171398360932, "rougeLsum_fmeasure": 0.12442129008035598, "rougeLsum_fmeasure_stderr": 0.0015468255706592889, "rougeLsum_precision": 0.14788595860466908, "rougeLsum_precision_stderr": 0.0022677650604797407, "rougeLsum_recall": 0.1476217188383832, "rougeLsum_recall_stderr": 0.002589681176386657}}, "2": {"PALM_prompt": {"bleu": 0.12302264952010052, "bleu_stderr": 0.008473608033646678, "rouge1_fmeasure": 0.0768642756778079, "rouge1_fmeasure_stderr": 0.001811200981366701, "rouge1_precision": 0.055679194052293414, "rouge1_precision_stderr": 0.0015790115772431718, "rouge1_recall": 0.1808141666654767, "rouge1_recall_stderr": 0.003768554209591496, "rouge2_fmeasure": 0.030198905160389044, "rouge2_fmeasure_stderr": 0.000982928967923372, "rouge2_precision": 0.02070146702208313, "rouge2_precision_stderr": 0.0007492369519392376, "rouge2_recall": 0.07660233137719231, "rouge2_recall_stderr": 0.00242307758789654, "rougeL_fmeasure": 0.07286820570794587, "rougeL_fmeasure_stderr": 0.0016717956031784844, "rougeL_precision": 0.052039959674758364, "rougeL_precision_stderr": 0.001392609794321448, "rougeL_recall": 0.17496136681829913, "rougeL_recall_stderr": 0.003685600912041933, "rougeLsum_fmeasure": 0.07325502354058079, "rougeLsum_fmeasure_stderr": 0.001694053285569078, "rougeLsum_precision": 0.052645110361589434, "rougeLsum_precision_stderr": 0.001455115958473737, "rougeLsum_recall": 0.17488088925366627, "rougeLsum_recall_stderr": 0.003655243712914895}, "explicit-graph-description2": {"bleu": 0.5492689650034025, "bleu_stderr": 0.08789814958865551, "rouge1_fmeasure": 0.14354986777463272, "rouge1_fmeasure_stderr": 0.0018339278799392887, "rouge1_precision": 0.1705969694142156, "rouge1_precision_stderr": 0.0025131229098136585, "rouge1_recall": 0.15761321437356865, "rouge1_recall_stderr": 0.002432157802246827, "rouge2_fmeasure": 0.014100180203474038, "rouge2_fmeasure_stderr": 0.0008276615210277857, "rouge2_precision": 0.017122190675118196, "rouge2_precision_stderr": 0.0012447818915600876, "rouge2_recall": 0.016763144168686534, "rouge2_recall_stderr": 0.001041171249249909, "rougeL_fmeasure": 0.11949064473731222, "rougeL_fmeasure_stderr": 0.001470601686701589, "rougeL_precision": 0.14270089197334904, "rougeL_precision_stderr": 0.002135234458917749, "rougeL_recall": 0.1326975552677369, "rougeL_recall_stderr": 0.0020774353171695834, "rougeLsum_fmeasure": 0.12656168844098892, "rougeLsum_fmeasure_stderr": 0.0015798154653780215, "rougeLsum_precision": 0.1515024612128631, "rougeLsum_precision_stderr": 0.002270510371981427, "rougeLsum_recall": 0.1392146312336204, "rougeLsum_recall_stderr": 0.002115600656748789}, "implicit-graph-description": {"bleu": 0.44030050792600856, "bleu_stderr": 0.08918965531864674, "rouge1_fmeasure": 0.14063207331011088, "rouge1_fmeasure_stderr": 0.0018732758899568784, "rouge1_precision": 0.165020909705563, "rouge1_precision_stderr": 0.002497799117783653, "rouge1_recall": 0.1579071204222324, "rouge1_recall_stderr": 0.0024816328723868283, "rouge2_fmeasure": 0.013404518942651407, "rouge2_fmeasure_stderr": 0.0007787079973307792, "rouge2_precision": 0.015020754513726486, "rouge2_precision_stderr": 0.0009072131098635255, "rouge2_recall": 0.016692167939649796, "rouge2_recall_stderr": 0.0010286596188788658, "rougeL_fmeasure": 0.11682645738111967, "rougeL_fmeasure_stderr": 0.0014963127749698607, "rougeL_precision": 0.13749360940884617, "rougeL_precision_stderr": 0.0020803253174644953, "rougeL_recall": 0.13328479413864802, "rougeL_recall_stderr": 0.002139290960697942, "rougeLsum_fmeasure": 0.1239705878552023, "rougeLsum_fmeasure_stderr": 0.0016213406765116238, "rougeLsum_precision": 0.1464246392027477, "rougeLsum_precision_stderr": 0.0022481093268697607, "rougeLsum_recall": 0.1390216810249052, "rougeLsum_recall_stderr": 0.0021491921792093326}, "non-explicit-description": {"bleu": 0.464666510911833, "bleu_stderr": 0.08222220573011234, "rouge1_fmeasure": 0.13892759709779767, "rouge1_fmeasure_stderr": 0.0018265728869274942, "rouge1_precision": 0.16319819867642651, "rouge1_precision_stderr": 0.002426464101736119, "rouge1_recall": 0.15404042754151479, "rouge1_recall_stderr": 0.002478306564411953, "rouge2_fmeasure": 0.013089719060620607, "rouge2_fmeasure_stderr": 0.0007726454211608279, "rouge2_precision": 0.014912982683458914, "rouge2_precision_stderr": 0.0009237093791681929, "rouge2_recall": 0.016297943525760927, "rouge2_recall_stderr": 0.0010649358289921388, "rougeL_fmeasure": 0.11650832190139555, "rougeL_fmeasure_stderr": 0.0014759630408007285, "rougeL_precision": 0.13741035910079322, "rougeL_precision_stderr": 0.0020428354842843533, "rougeL_recall": 0.13057150223971264, "rougeL_recall_stderr": 0.0021219065821278115, "rougeLsum_fmeasure": 0.1229747664140421, "rougeLsum_fmeasure_stderr": 0.0015963298971135365, "rougeLsum_precision": 0.14553524668113407, "rougeLsum_precision_stderr": 0.0022002606329358026, "rougeLsum_recall": 0.13620255058111988, "rougeLsum_recall_stderr": 0.0021639599169587793}, "very-explicit-description": {"bleu": 0.5931779549207392, "bleu_stderr": 0.09339568633223094, "rouge1_fmeasure": 0.14125838813948505, "rouge1_fmeasure_stderr": 0.0018147591314905, "rouge1_precision": 0.1653712229197853, "rouge1_precision_stderr": 0.002440924984163296, "rouge1_recall": 0.1653250915896547, "rouge1_recall_stderr": 0.0029135773698179563, "rouge2_fmeasure": 0.014494882572376312, "rouge2_fmeasure_stderr": 0.0008275950679432722, "rouge2_precision": 0.01650829521165785, "rouge2_precision_stderr": 0.0010046369743166356, "rouge2_recall": 0.020600281994805424, "rouge2_recall_stderr": 0.0013239499542895858, "rougeL_fmeasure": 0.1184115219641423, "rougeL_fmeasure_stderr": 0.00146376498152489, "rougeL_precision": 0.13917888276805995, "rougeL_precision_stderr": 0.0020659023219378315, "rougeL_recall": 0.14043061866293993, "rougeL_recall_stderr": 0.002517638410671949, "rougeLsum_fmeasure": 0.1247729325662712, "rougeLsum_fmeasure_stderr": 0.001572783517385721, "rougeLsum_precision": 0.1471096818527704, "rougeLsum_precision_stderr": 0.0022017203919913464, "rougeLsum_recall": 0.14601316919655402, "rougeLsum_recall_stderr": 0.0025452049628687883}}, "3": {"PALM_prompt": {"bleu": 0.1494775278828044, "bleu_stderr": 0.022242668837387642, "rouge1_fmeasure": 0.07759644830725707, "rouge1_fmeasure_stderr": 0.0017814677119907972, "rouge1_precision": 0.05819590767349479, "rouge1_precision_stderr": 0.00187266941961563, "rouge1_recall": 0.1837211769068863, "rouge1_recall_stderr": 0.0037631739340242853, "rouge2_fmeasure": 0.0299181905955771, "rouge2_fmeasure_stderr": 0.0009706547160371815, "rouge2_precision": 0.020971065102809404, "rouge2_precision_stderr": 0.000895456781665476, "rouge2_recall": 0.078084178799313, "rouge2_recall_stderr": 0.0024146049927506823, "rougeL_fmeasure": 0.07316137688753947, "rougeL_fmeasure_stderr": 0.0016471496270948857, "rougeL_precision": 0.05435903919389146, "rougeL_precision_stderr": 0.001720611632601612, "rougeL_recall": 0.17679767348251002, "rougeL_recall_stderr": 0.0036809106852510385, "rougeLsum_fmeasure": 0.07366095409257907, "rougeLsum_fmeasure_stderr": 0.0016586575850230274, "rougeLsum_precision": 0.05492316378292939, "rougeLsum_precision_stderr": 0.001744093415859728, "rougeLsum_recall": 0.17690363183304764, "rougeLsum_recall_stderr": 0.003638138098577823}, "explicit-graph-description2": {"bleu": 0.5225086157392205, "bleu_stderr": 0.08782464964651336, "rouge1_fmeasure": 0.1441015790918314, "rouge1_fmeasure_stderr": 0.0018602827624463706, "rouge1_precision": 0.16990770521290732, "rouge1_precision_stderr": 0.0025438057474100073, "rouge1_recall": 0.16021997486913941, "rouge1_recall_stderr": 0.0024160108734416854, "rouge2_fmeasure": 0.013249922136095603, "rouge2_fmeasure_stderr": 0.0008649946361473154, "rouge2_precision": 0.015214536900019273, "rouge2_precision_stderr": 0.0010692205053183784, "rouge2_recall": 0.015420185258765253, "rouge2_recall_stderr": 0.0010273736824902339, "rougeL_fmeasure": 0.11986132557556263, "rougeL_fmeasure_stderr": 0.001509826051165344, "rougeL_precision": 0.14196899698534282, "rougeL_precision_stderr": 0.0021319224801637135, "rougeL_recall": 0.13510011259495838, "rougeL_recall_stderr": 0.0020902509105043673, "rougeLsum_fmeasure": 0.1268605469148488, "rougeLsum_fmeasure_stderr": 0.0016208332554181627, "rougeLsum_precision": 0.15080440042366344, "rougeLsum_precision_stderr": 0.0022898520672142444, "rougeLsum_recall": 0.14098767187068265, "rougeLsum_recall_stderr": 0.0021167327935460206}, "implicit-graph-description": {"bleu": 0.4028004322246814, "bleu_stderr": 0.09627992264831524, "rouge1_fmeasure": 0.13807126726930546, "rouge1_fmeasure_stderr": 0.001961867694651278, "rouge1_precision": 0.16192215223479117, "rouge1_precision_stderr": 0.0025886791288695856, "rouge1_recall": 0.1591737175072444, "rouge1_recall_stderr": 0.002676280527776092, "rouge2_fmeasure": 0.013694547280363368, "rouge2_fmeasure_stderr": 0.0008406341572135305, "rouge2_precision": 0.01532497701640921, "rouge2_precision_stderr": 0.00096053634684961, "rouge2_recall": 0.01789090329021704, "rouge2_recall_stderr": 0.001138291110706073, "rougeL_fmeasure": 0.11455050831050843, "rougeL_fmeasure_stderr": 0.001572738062460073, "rougeL_precision": 0.1348196406800519, "rougeL_precision_stderr": 0.0021490241206731956, "rougeL_recall": 0.13491070126011628, "rougeL_recall_stderr": 0.0023624548127525523, "rougeLsum_fmeasure": 0.12128174476152666, "rougeLsum_fmeasure_stderr": 0.0017094370940622802, "rougeLsum_precision": 0.1434457740106499, "rougeLsum_precision_stderr": 0.0023224644970998463, "rougeLsum_recall": 0.13929199647413618, "rougeLsum_recall_stderr": 0.00233665463674003}, "non-explicit-description": {"bleu": 0.5347923980216038, "bleu_stderr": 0.05654660025438367, "rouge1_fmeasure": 0.1351641368693967, "rouge1_fmeasure_stderr": 0.0019474873765221823, "rouge1_precision": 0.16188795072384957, "rouge1_precision_stderr": 0.0025606178815772186, "rouge1_recall": 0.14779323773320033, "rouge1_recall_stderr": 0.0025010089208298046, "rouge2_fmeasure": 0.013153025539986579, "rouge2_fmeasure_stderr": 0.0008953407702700355, "rouge2_precision": 0.015524787502426208, "rouge2_precision_stderr": 0.001042198532546253, "rouge2_recall": 0.015185223617908139, "rouge2_recall_stderr": 0.00114853509785443, "rougeL_fmeasure": 0.11226025544826589, "rougeL_fmeasure_stderr": 0.0015601459007713312, "rougeL_precision": 0.13524306101257844, "rougeL_precision_stderr": 0.002145203133780158, "rougeL_recall": 0.12387529380771088, "rougeL_recall_stderr": 0.002083956552126564, "rougeLsum_fmeasure": 0.11871298212567302, "rougeLsum_fmeasure_stderr": 0.0016806710065392066, "rougeLsum_precision": 0.14350630370022732, "rougeLsum_precision_stderr": 0.0022925759452389416, "rougeLsum_recall": 0.12941050420184938, "rougeLsum_recall_stderr": 0.002157229969105682}, "very-explicit-description": {"bleu": 0.613415825073411, "bleu_stderr": 0.15363249845233518, "rouge1_fmeasure": 0.13960615389268183, "rouge1_fmeasure_stderr": 0.0017523193011045669, "rouge1_precision": 0.16582665337512495, "rouge1_precision_stderr": 0.0024208349959723994, "rouge1_recall": 0.16255706599105907, "rouge1_recall_stderr": 0.002834223972206501, "rouge2_fmeasure": 0.013187050625056523, "rouge2_fmeasure_stderr": 0.0007211908914592551, "rouge2_precision": 0.015177432476958089, "rouge2_precision_stderr": 0.0008927376595589448, "rouge2_recall": 0.019082870994391096, "rouge2_recall_stderr": 0.001300150390527605, "rougeL_fmeasure": 0.11581175106585567, "rougeL_fmeasure_stderr": 0.0013986409084470917, "rougeL_precision": 0.13831816790209567, "rougeL_precision_stderr": 0.0020310042200119573, "rougeL_recall": 0.13727289920306487, "rougeL_recall_stderr": 0.0025055915595956126, "rougeLsum_fmeasure": 0.12243760940103116, "rougeLsum_fmeasure_stderr": 0.0014992182881352384, "rougeLsum_precision": 0.14670366020679268, "rougeLsum_precision_stderr": 0.00216023102430518, "rougeLsum_recall": 0.1428602401569226, "rougeLsum_recall_stderr": 0.002485596883393547}}, "4": {"PALM_prompt": {"bleu": 0.19164606319365804, "bleu_stderr": 0.024842965498983755, "rouge1_fmeasure": 0.08218073938962703, "rouge1_fmeasure_stderr": 0.0018499322659086194, "rouge1_precision": 0.05993715403649896, "rouge1_precision_stderr": 0.0016748088373488277, "rouge1_recall": 0.18999825922564922, "rouge1_recall_stderr": 0.0037155688461071694, "rouge2_fmeasure": 0.031639571635299846, "rouge2_fmeasure_stderr": 0.0009862002820496016, "rouge2_precision": 0.021877668010459907, "rouge2_precision_stderr": 0.0007551954975307327, "rouge2_recall": 0.07987598814280476, "rouge2_recall_stderr": 0.0023972171750080043, "rougeL_fmeasure": 0.07749057699272385, "rougeL_fmeasure_stderr": 0.0016921799932615024, "rougeL_precision": 0.05569962429765539, "rougeL_precision_stderr": 0.0014542809726584018, "rougeL_recall": 0.1829897393194941, "rougeL_recall_stderr": 0.003626975042403296, "rougeLsum_fmeasure": 0.07799095859947036, "rougeLsum_fmeasure_stderr": 0.0017129521717618506, "rougeLsum_precision": 0.056374413163303896, "rougeLsum_precision_stderr": 0.0015045460779383418, "rougeLsum_recall": 0.1831731672325477, "rougeLsum_recall_stderr": 0.003597143627516616}, "explicit-graph-description2": {"bleu": 0.5342732499256956, "bleu_stderr": 0.1027315904453386, "rouge1_fmeasure": 0.14606482836681436, "rouge1_fmeasure_stderr": 0.0018280556446259094, "rouge1_precision": 0.17001348002992608, "rouge1_precision_stderr": 0.002472945726795613, "rouge1_recall": 0.16641335439404173, "rouge1_recall_stderr": 0.002516334305490709, "rouge2_fmeasure": 0.014229702953664455, "rouge2_fmeasure_stderr": 0.0008237983700670122, "rouge2_precision": 0.01591640786398681, "rouge2_precision_stderr": 0.000961538418662567, "rouge2_recall": 0.017920512300572296, "rouge2_recall_stderr": 0.001111692174103912, "rougeL_fmeasure": 0.12175024842032746, "rougeL_fmeasure_stderr": 0.0014788274431685848, "rougeL_precision": 0.14223869415288778, "rougeL_precision_stderr": 0.0020766661579404897, "rougeL_recall": 0.14080246405083494, "rougeL_recall_stderr": 0.0021890583449121997, "rougeLsum_fmeasure": 0.12836641711166125, "rougeLsum_fmeasure_stderr": 0.0015963226153607253, "rougeLsum_precision": 0.1506981914188859, "rougeLsum_precision_stderr": 0.00224238363239455, "rougeLsum_recall": 0.14600473949937198, "rougeLsum_recall_stderr": 0.0021735849443758988}, "implicit-graph-description": {"bleu": 0.48038870322086485, "bleu_stderr": 0.0950744978183185, "rouge1_fmeasure": 0.13898420590680097, "rouge1_fmeasure_stderr": 0.002005636475842531, "rouge1_precision": 0.1612689936412253, "rouge1_precision_stderr": 0.002593706132714892, "rouge1_recall": 0.16230890102132264, "rouge1_recall_stderr": 0.0027329862790525937, "rouge2_fmeasure": 0.01467598496147871, "rouge2_fmeasure_stderr": 0.000943689965606244, "rouge2_precision": 0.01639943402464418, "rouge2_precision_stderr": 0.0010462999278646942, "rouge2_recall": 0.019047005353722873, "rouge2_recall_stderr": 0.0012346130129613677, "rougeL_fmeasure": 0.11632551379631219, "rougeL_fmeasure_stderr": 0.0016469373399055434, "rougeL_precision": 0.13528835328741515, "rougeL_precision_stderr": 0.0021828816198137323, "rougeL_recall": 0.1384797370898447, "rougeL_recall_stderr": 0.0024281776883333106, "rougeLsum_fmeasure": 0.12238496569136655, "rougeLsum_fmeasure_stderr": 0.0017501406428895951, "rougeLsum_precision": 0.1433977750531889, "rougeLsum_precision_stderr": 0.0023475946507656165, "rougeLsum_recall": 0.14234860995071957, "rougeLsum_recall_stderr": 0.0023567460311511336}, "non-explicit-description": {"bleu": 0.447830278815385, "bleu_stderr": 0.06715101669775739, "rouge1_fmeasure": 0.13890191242592734, "rouge1_fmeasure_stderr": 0.0018984947282986685, "rouge1_precision": 0.1643685889453247, "rouge1_precision_stderr": 0.0024901757152926327, "rouge1_recall": 0.15465227513600494, "rouge1_recall_stderr": 0.0025856665354129014, "rouge2_fmeasure": 0.013872966659414053, "rouge2_fmeasure_stderr": 0.0007622040129882122, "rouge2_precision": 0.015811707006913117, "rouge2_precision_stderr": 0.0008876824189595942, "rouge2_recall": 0.017023747384715765, "rouge2_recall_stderr": 0.001064465606217159, "rougeL_fmeasure": 0.11546828539245778, "rougeL_fmeasure_stderr": 0.0015269267163462042, "rougeL_precision": 0.13761313588982987, "rougeL_precision_stderr": 0.002106009553028678, "rougeL_recall": 0.1298443924561633, "rougeL_recall_stderr": 0.002193510884352127, "rougeLsum_fmeasure": 0.12188337669823847, "rougeLsum_fmeasure_stderr": 0.0016518640850592327, "rougeLsum_precision": 0.1458255331490558, "rougeLsum_precision_stderr": 0.0022720293179680218, "rougeLsum_recall": 0.13542456041940654, "rougeLsum_recall_stderr": 0.0022271994295923335}, "very-explicit-description": {"bleu": 0.5374477526028321, "bleu_stderr": 0.09283692815634105, "rouge1_fmeasure": 0.1415749443824625, "rouge1_fmeasure_stderr": 0.0018144061370138271, "rouge1_precision": 0.16440247158022403, "rouge1_precision_stderr": 0.002408134958644391, "rouge1_recall": 0.1672878148244216, "rouge1_recall_stderr": 0.0028544677726046685, "rouge2_fmeasure": 0.014802883130151939, "rouge2_fmeasure_stderr": 0.0007502428586822482, "rouge2_precision": 0.016582657589509987, "rouge2_precision_stderr": 0.0008859080182706377, "rouge2_recall": 0.02110856663946593, "rouge2_recall_stderr": 0.001326962436284589, "rougeL_fmeasure": 0.11783817259815357, "rougeL_fmeasure_stderr": 0.001441797703688301, "rougeL_precision": 0.13771889673883428, "rougeL_precision_stderr": 0.002017980024471294, "rougeL_recall": 0.14130600028610976, "rougeL_recall_stderr": 0.0024909401470214177, "rougeLsum_fmeasure": 0.12419545819826318, "rougeLsum_fmeasure_stderr": 0.0015690813210837236, "rougeLsum_precision": 0.14564352836992275, "rougeLsum_precision_stderr": 0.002182392381771102, "rougeLsum_recall": 0.14647550822693875, "rougeLsum_recall_stderr": 0.002455629059429704}}, "5": {"PALM_prompt": {"bleu": 0.17574933687846614, "bleu_stderr": 0.025384181870486397, "rouge1_fmeasure": 0.07946226594400384, "rouge1_fmeasure_stderr": 0.0018412019107524114, "rouge1_precision": 0.05972888483104343, "rouge1_precision_stderr": 0.0018329849074038712, "rouge1_recall": 0.18292682744906683, "rouge1_recall_stderr": 0.003663997330795685, "rouge2_fmeasure": 0.030022553056029706, "rouge2_fmeasure_stderr": 0.0009742418086937908, "rouge2_precision": 0.021074770698868366, "rouge2_precision_stderr": 0.0007683455677750384, "rouge2_recall": 0.07513861469355282, "rouge2_recall_stderr": 0.002333692457604091, "rougeL_fmeasure": 0.07483720907439476, "rougeL_fmeasure_stderr": 0.001672817823692251, "rougeL_precision": 0.05516127047546632, "rougeL_precision_stderr": 0.0015392382782714019, "rougeL_recall": 0.17614313937858433, "rougeL_recall_stderr": 0.0035646877950734107, "rougeLsum_fmeasure": 0.07489823136944627, "rougeLsum_fmeasure_stderr": 0.0016865681248032596, "rougeLsum_precision": 0.055368021748712734, "rougeLsum_precision_stderr": 0.001561582516111846, "rougeLsum_recall": 0.1756797982007548, "rougeLsum_recall_stderr": 0.003531535198778665}, "explicit-graph-description2": {"bleu": 0.5599756287973008, "bleu_stderr": 0.08624872080359487, "rouge1_fmeasure": 0.14725933348138234, "rouge1_fmeasure_stderr": 0.0018907376621132115, "rouge1_precision": 0.17335244138244874, "rouge1_precision_stderr": 0.002628810045816908, "rouge1_recall": 0.16573763442748227, "rouge1_recall_stderr": 0.002438138421190895, "rouge2_fmeasure": 0.015163133405164434, "rouge2_fmeasure_stderr": 0.0009412775453260759, "rouge2_precision": 0.018051903585621448, "rouge2_precision_stderr": 0.0011163727106672703, "rouge2_recall": 0.017828395181452815, "rouge2_recall_stderr": 0.0011751907098589418, "rougeL_fmeasure": 0.12234405519301784, "rougeL_fmeasure_stderr": 0.00149721265164285, "rougeL_precision": 0.1443998895689471, "rougeL_precision_stderr": 0.0021924349422918392, "rougeL_recall": 0.13967945448595898, "rougeL_recall_stderr": 0.002066115416965032, "rougeLsum_fmeasure": 0.12944852616669666, "rougeLsum_fmeasure_stderr": 0.001605797602788086, "rougeLsum_precision": 0.15358706687881649, "rougeLsum_precision_stderr": 0.0023486595525211425, "rougeLsum_recall": 0.14542801653317822, "rougeLsum_recall_stderr": 0.0020859852388272707}, "implicit-graph-description": {"bleu": 0.45115235727169706, "bleu_stderr": 0.07183544899400104, "rouge1_fmeasure": 0.14037246602351963, "rouge1_fmeasure_stderr": 0.002008576466387765, "rouge1_precision": 0.16524773113217792, "rouge1_precision_stderr": 0.002761577061548867, "rouge1_recall": 0.1671900838419029, "rouge1_recall_stderr": 0.0028680251377737, "rouge2_fmeasure": 0.01573629832661093, "rouge2_fmeasure_stderr": 0.0009296010590017547, "rouge2_precision": 0.01852378604139675, "rouge2_precision_stderr": 0.001108714616606197, "rouge2_recall": 0.02208117704754726, "rouge2_recall_stderr": 0.0014164098810313566, "rougeL_fmeasure": 0.1174214317896419, "rougeL_fmeasure_stderr": 0.0016188198074821145, "rougeL_precision": 0.13842446049242282, "rougeL_precision_stderr": 0.0023215511141633505, "rougeL_recall": 0.14325804112156512, "rougeL_recall_stderr": 0.002575571482543266, "rougeLsum_fmeasure": 0.12334767454389015, "rougeLsum_fmeasure_stderr": 0.001735345048925764, "rougeLsum_precision": 0.14624932204164548, "rougeLsum_precision_stderr": 0.0024721564063546987, "rougeLsum_recall": 0.14630209531166793, "rougeLsum_recall_stderr": 0.002457955701083795}, "non-explicit-description": {"bleu": 0.4758464184876175, "bleu_stderr": 0.10146555413589751, "rouge1_fmeasure": 0.13625801392457523, "rouge1_fmeasure_stderr": 0.00187042901923777, "rouge1_precision": 0.1646198127817207, "rouge1_precision_stderr": 0.002646484323779933, "rouge1_recall": 0.1490304755831364, "rouge1_recall_stderr": 0.0023689773981536632, "rouge2_fmeasure": 0.013595010492421218, "rouge2_fmeasure_stderr": 0.0007463780707733565, "rouge2_precision": 0.01713122779441091, "rouge2_precision_stderr": 0.0010058504976427649, "rouge2_recall": 0.015592738904389145, "rouge2_recall_stderr": 0.0009494920127045856, "rougeL_fmeasure": 0.11419438860339491, "rougeL_fmeasure_stderr": 0.0015164933018915136, "rougeL_precision": 0.13886708223114388, "rougeL_precision_stderr": 0.002259848022664383, "rougeL_recall": 0.12602515814645163, "rougeL_recall_stderr": 0.0019898391778165677, "rougeLsum_fmeasure": 0.12064027783181062, "rougeLsum_fmeasure_stderr": 0.001640982408038248, "rougeLsum_precision": 0.14703846057442585, "rougeLsum_precision_stderr": 0.0024248196702567554, "rougeLsum_recall": 0.13191735359035292, "rougeLsum_recall_stderr": 0.0020719729544053833}, "very-explicit-description": {"bleu": 0.5504965404148479, "bleu_stderr": 0.080427930138377, "rouge1_fmeasure": 0.14047911280614023, "rouge1_fmeasure_stderr": 0.001918909621032034, "rouge1_precision": 0.16597024457966064, "rouge1_precision_stderr": 0.0026243195088935236, "rouge1_recall": 0.16986441790095472, "rouge1_recall_stderr": 0.0030299534520702256, "rouge2_fmeasure": 0.01624180543052662, "rouge2_fmeasure_stderr": 0.0009504654937149193, "rouge2_precision": 0.018689456645170934, "rouge2_precision_stderr": 0.001121139158372213, "rouge2_recall": 0.023799423953292702, "rouge2_recall_stderr": 0.0015503997097516637, "rougeL_fmeasure": 0.11793359101554969, "rougeL_fmeasure_stderr": 0.0015470077154010478, "rougeL_precision": 0.13981148514085578, "rougeL_precision_stderr": 0.002224344509680242, "rougeL_recall": 0.14559045715327285, "rougeL_recall_stderr": 0.0026961659834251164, "rougeLsum_fmeasure": 0.12410304209348472, "rougeLsum_fmeasure_stderr": 0.0016642202118066153, "rougeLsum_precision": 0.14803607615354483, "rougeLsum_precision_stderr": 0.0023878068281125613, "rougeLsum_recall": 0.14963478549748452, "rougeLsum_recall_stderr": 0.0026025633576893647}}}, "GEM/wiki_lingua_en": {"0": {"article_summary_en": {"bleu": 2.3244260259071865, "bleu_stderr": 0.07286281897100536, "rouge1_fmeasure": 0.22563848251054694, "rouge1_fmeasure_stderr": 0.0018751473618148513, "rouge1_precision": 0.1890231240274845, "rouge1_precision_stderr": 0.0019684320777096703, "rouge1_recall": 0.339017718024998, "rouge1_recall_stderr": 0.0028323433133959483, "rouge2_fmeasure": 0.052567975188848405, "rouge2_fmeasure_stderr": 0.0009556547743829917, "rouge2_precision": 0.04380838750833082, "rouge2_precision_stderr": 0.0008419429245577659, "rouge2_recall": 0.08179849168709151, "rouge2_recall_stderr": 0.0017042047474535047, "rougeL_fmeasure": 0.15451828388176664, "rougeL_fmeasure_stderr": 0.001208772883015503, "rougeL_precision": 0.127676216716478, "rougeL_precision_stderr": 0.0012153296324758198, "rougeL_recall": 0.23953125612828238, "rougeL_recall_stderr": 0.0022402139432793856, "rougeLsum_fmeasure": 0.20883538497621398, "rougeLsum_fmeasure_stderr": 0.0017334509196587264, "rougeLsum_precision": 0.17481696246700595, "rougeLsum_precision_stderr": 0.0018160527235596866, "rougeLsum_recall": 0.31443864114060294, "rougeLsum_recall_stderr": 0.0026582156582863913}, "rephrase_en": {"bleu": 0.4367609232393782, "bleu_stderr": 0.023336487096873623, "rouge1_fmeasure": 0.07728270900364194, "rouge1_fmeasure_stderr": 0.001418281974629439, "rouge1_precision": 0.07162315852179309, "rouge1_precision_stderr": 0.0014555411662114063, "rouge1_recall": 0.1024819369868203, "rouge1_recall_stderr": 0.0019137149698756734, "rouge2_fmeasure": 0.007854289251701565, "rouge2_fmeasure_stderr": 0.00040869938896478014, "rouge2_precision": 0.007210468394688485, "rouge2_precision_stderr": 0.00037775068659526225, "rouge2_recall": 0.010657999963685463, "rouge2_recall_stderr": 0.000602849822298443, "rougeL_fmeasure": 0.06907689906047816, "rougeL_fmeasure_stderr": 0.0011853028745420664, "rougeL_precision": 0.06359760111567767, "rougeL_precision_stderr": 0.0012025425436729914, "rougeL_recall": 0.09274514173812211, "rougeL_recall_stderr": 0.0016759276752536403, "rougeLsum_fmeasure": 0.07283735339621857, "rougeLsum_fmeasure_stderr": 0.0013034613782005062, "rougeLsum_precision": 0.06737233028287096, "rougeLsum_precision_stderr": 0.0013379489246377642, "rougeLsum_recall": 0.09693941896958716, "rougeLsum_recall_stderr": 0.0017781152777598497}, "summarize_above_en": {"bleu": 0.2645947287972745, "bleu_stderr": 0.021907771840493258, "rouge1_fmeasure": 0.050630622435102954, "rouge1_fmeasure_stderr": 0.0012740642440480783, "rouge1_precision": 0.04418667472732478, "rouge1_precision_stderr": 0.0012197029509449378, "rouge1_recall": 0.07215894137790155, "rouge1_recall_stderr": 0.0017665079751954936, "rouge2_fmeasure": 0.0038458659144674466, "rouge2_fmeasure_stderr": 0.0003374984459731064, "rouge2_precision": 0.0033950280387810075, "rouge2_precision_stderr": 0.0003033243161776345, "rouge2_recall": 0.005381920499729423, "rouge2_recall_stderr": 0.00048448492307215835, "rougeL_fmeasure": 0.045212023379810784, "rougeL_fmeasure_stderr": 0.0010775987760050663, "rougeL_precision": 0.03927568004870594, "rougeL_precision_stderr": 0.0010304820118418286, "rougeL_recall": 0.0651799283564036, "rougeL_recall_stderr": 0.0015341625257454653, "rougeLsum_fmeasure": 0.047792067497451136, "rougeLsum_fmeasure_stderr": 0.001175048993772459, "rougeLsum_precision": 0.04166366640338202, "rougeLsum_precision_stderr": 0.001132597373996381, "rougeLsum_recall": 0.06842472788539408, "rougeLsum_recall_stderr": 0.0016398413239512239}, "tldr_en": {"bleu": 0.9033237981203045, "bleu_stderr": 0.06093048704966743, "rouge1_fmeasure": 0.09664038969174059, "rouge1_fmeasure_stderr": 0.0018523781781760917, "rouge1_precision": 0.08475422777270436, "rouge1_precision_stderr": 0.0017956983368717673, "rouge1_recall": 0.136466539852913, "rouge1_recall_stderr": 0.0026047032098612134, "rouge2_fmeasure": 0.015671748077472996, "rouge2_fmeasure_stderr": 0.0006102024889325685, "rouge2_precision": 0.013532165780802595, "rouge2_precision_stderr": 0.0005500871126004836, "rouge2_recall": 0.023126949071911686, "rouge2_recall_stderr": 0.000984728105845212, "rougeL_fmeasure": 0.08141292643254222, "rougeL_fmeasure_stderr": 0.0014662345345116555, "rougeL_precision": 0.07093659031967181, "rougeL_precision_stderr": 0.001414812786895555, "rougeL_recall": 0.11686852788639902, "rougeL_recall_stderr": 0.0021776574996213755, "rougeLsum_fmeasure": 0.09028162427912649, "rougeLsum_fmeasure_stderr": 0.0017170249037180556, "rougeLsum_precision": 0.07902371474904281, "rougeLsum_precision_stderr": 0.001655136022979042, "rougeLsum_recall": 0.12774835789500472, "rougeLsum_recall_stderr": 0.0024261371686350093}, "write_abstract_en": {"bleu": 0.745448160166905, "bleu_stderr": 0.047547732193019436, "rouge1_fmeasure": 0.10023372190565902, "rouge1_fmeasure_stderr": 0.0017262119489058103, "rouge1_precision": 0.09246228962154374, "rouge1_precision_stderr": 0.001943807421175789, "rouge1_recall": 0.13805949306072055, "rouge1_recall_stderr": 0.002409786846441639, "rouge2_fmeasure": 0.014303497306763413, "rouge2_fmeasure_stderr": 0.0006015388106615314, "rouge2_precision": 0.012545852280114898, "rouge2_precision_stderr": 0.0005482585896654373, "rouge2_recall": 0.021241128219617465, "rouge2_recall_stderr": 0.001043761840842228, "rougeL_fmeasure": 0.08504732396245059, "rougeL_fmeasure_stderr": 0.001313235354820681, "rougeL_precision": 0.07837678044248741, "rougeL_precision_stderr": 0.001609888637751305, "rougeL_recall": 0.11893781436000582, "rougeL_recall_stderr": 0.0019490781148502943, "rougeLsum_fmeasure": 0.09216848394827927, "rougeLsum_fmeasure_stderr": 0.001563789121056033, "rougeLsum_precision": 0.08436461190041952, "rougeLsum_precision_stderr": 0.0017072730429511782, "rougeLsum_recall": 0.12779895161379792, "rougeLsum_recall_stderr": 0.0022186936931219443}}, "1": {"article_summary_en": {"bleu": 1.828441133091315, "bleu_stderr": 0.04797244163227974, "rouge1_fmeasure": 0.18474230303970723, "rouge1_fmeasure_stderr": 0.0019632536598955266, "rouge1_precision": 0.16030602957044252, "rouge1_precision_stderr": 0.001967524581211362, "rouge1_recall": 0.272400210100343, "rouge1_recall_stderr": 0.003051643168753336, "rouge2_fmeasure": 0.03761257717991928, "rouge2_fmeasure_stderr": 0.0008913626176034577, "rouge2_precision": 0.03166192462922729, "rouge2_precision_stderr": 0.0007933357990075433, "rouge2_recall": 0.05886551805358644, "rouge2_recall_stderr": 0.0015884469531505948, "rougeL_fmeasure": 0.13089239358773688, "rougeL_fmeasure_stderr": 0.0012729550476269687, "rougeL_precision": 0.11291539782776717, "rougeL_precision_stderr": 0.0012748009948283148, "rougeL_recall": 0.19815104405287912, "rougeL_recall_stderr": 0.002316563028814676, "rougeLsum_fmeasure": 0.1719442252457088, "rougeLsum_fmeasure_stderr": 0.001807476862548232, "rougeLsum_precision": 0.14915402681793893, "rougeLsum_precision_stderr": 0.0018104204962086678, "rougeLsum_recall": 0.25378286760463403, "rougeLsum_recall_stderr": 0.0028305419282366193}, "rephrase_en": {"bleu": 0.33862715230568985, "bleu_stderr": 0.02103608050658023, "rouge1_fmeasure": 0.09906222042576335, "rouge1_fmeasure_stderr": 0.0012277637555703933, "rouge1_precision": 0.10962094154854289, "rouge1_precision_stderr": 0.001521175658021088, "rouge1_recall": 0.11614995439958446, "rouge1_recall_stderr": 0.0016222282716261732, "rouge2_fmeasure": 0.004986359493861402, "rouge2_fmeasure_stderr": 0.0002877838903421649, "rouge2_precision": 0.005375616655817727, "rouge2_precision_stderr": 0.00032065018661247955, "rouge2_recall": 0.005997760028548002, "rouge2_recall_stderr": 0.00040011451896884595, "rougeL_fmeasure": 0.07875322436499038, "rougeL_fmeasure_stderr": 0.000899325603615052, "rougeL_precision": 0.08788884421952135, "rougeL_precision_stderr": 0.0011845230529361117, "rougeL_recall": 0.0930440125082319, "rougeL_recall_stderr": 0.0012664554630627011, "rougeLsum_fmeasure": 0.09495364214558669, "rougeLsum_fmeasure_stderr": 0.001162653836627394, "rougeLsum_precision": 0.10514951471452734, "rougeLsum_precision_stderr": 0.001446840001157129, "rougeLsum_recall": 0.11139163899995776, "rougeLsum_recall_stderr": 0.0015440273459916564}, "summarize_above_en": {"bleu": 0.26189100567433143, "bleu_stderr": 0.04164591416025924, "rouge1_fmeasure": 0.09711652466721561, "rouge1_fmeasure_stderr": 0.00114856823101296, "rouge1_precision": 0.11065265004766811, "rouge1_precision_stderr": 0.0014995222918311947, "rouge1_recall": 0.10967982839934531, "rouge1_recall_stderr": 0.0014540559708165963, "rouge2_fmeasure": 0.0038867114952000873, "rouge2_fmeasure_stderr": 0.00023684798185580298, "rouge2_precision": 0.004476358135369021, "rouge2_precision_stderr": 0.00028847561060159696, "rouge2_recall": 0.0044039325000597256, "rouge2_recall_stderr": 0.0002972809876903113, "rougeL_fmeasure": 0.07784452056813264, "rougeL_fmeasure_stderr": 0.0008619620021596947, "rougeL_precision": 0.08933483124530466, "rougeL_precision_stderr": 0.0011884320054885904, "rougeL_recall": 0.08853836595721083, "rougeL_recall_stderr": 0.0011479891985354547, "rougeLsum_fmeasure": 0.09355863188288549, "rougeLsum_fmeasure_stderr": 0.0010951306163491682, "rougeLsum_precision": 0.10659695950864206, "rougeLsum_precision_stderr": 0.0014326181370186257, "rougeLsum_recall": 0.10579025460545817, "rougeLsum_recall_stderr": 0.001398522609494789}, "tldr_en": {"bleu": 0.48568473425226133, "bleu_stderr": 0.08021116263112237, "rouge1_fmeasure": 0.10140029000732263, "rouge1_fmeasure_stderr": 0.0012698513145956132, "rouge1_precision": 0.11341569440193765, "rouge1_precision_stderr": 0.0015875420885948467, "rouge1_recall": 0.11847927588394622, "rouge1_recall_stderr": 0.0017387628767267596, "rouge2_fmeasure": 0.005755944338368179, "rouge2_fmeasure_stderr": 0.00034584001151602705, "rouge2_precision": 0.006128471241361133, "rouge2_precision_stderr": 0.0003870105678849676, "rouge2_recall": 0.0076069170914782195, "rouge2_recall_stderr": 0.0005734677696249654, "rougeL_fmeasure": 0.08090174235356164, "rougeL_fmeasure_stderr": 0.0009344918578662129, "rougeL_precision": 0.09114212196429179, "rougeL_precision_stderr": 0.0012300275412546436, "rougeL_recall": 0.09521970565929594, "rougeL_recall_stderr": 0.0013695362241421447, "rougeLsum_fmeasure": 0.09730799225374498, "rougeLsum_fmeasure_stderr": 0.0011957026786911643, "rougeLsum_precision": 0.10896112722491796, "rougeLsum_precision_stderr": 0.0015092137229918339, "rougeLsum_recall": 0.11363652894302366, "rougeLsum_recall_stderr": 0.0016407183665521966}, "write_abstract_en": {"bleu": 0.36679033215823664, "bleu_stderr": 0.029277836361420107, "rouge1_fmeasure": 0.09978220637884393, "rouge1_fmeasure_stderr": 0.0012449757420128587, "rouge1_precision": 0.10990140516594758, "rouge1_precision_stderr": 0.0015351949382576508, "rouge1_recall": 0.11791561392844088, "rouge1_recall_stderr": 0.0016685835887549996, "rouge2_fmeasure": 0.0052645090649109895, "rouge2_fmeasure_stderr": 0.00031778114128576976, "rouge2_precision": 0.005589121653905529, "rouge2_precision_stderr": 0.0003414693806880958, "rouge2_recall": 0.006477748246242698, "rouge2_recall_stderr": 0.0004490780641964607, "rougeL_fmeasure": 0.07936011949636798, "rougeL_fmeasure_stderr": 0.0009144164542617104, "rougeL_precision": 0.08809126672816353, "rougeL_precision_stderr": 0.0011930762685352886, "rougeL_recall": 0.0946530891608615, "rougeL_recall_stderr": 0.0013251798935214189, "rougeLsum_fmeasure": 0.09589920116880182, "rougeLsum_fmeasure_stderr": 0.0011829464434838014, "rougeLsum_precision": 0.10565199584207204, "rougeLsum_precision_stderr": 0.0014615877820162168, "rougeLsum_recall": 0.11343373068042586, "rougeLsum_recall_stderr": 0.0015989418107090036}}, "2": {"article_summary_en": {"bleu": 1.319957729947558, "bleu_stderr": 0.055949786634926266, "rouge1_fmeasure": 0.1553013724184807, "rouge1_fmeasure_stderr": 0.001829739118077522, "rouge1_precision": 0.13691057056766903, "rouge1_precision_stderr": 0.001825265602905107, "rouge1_recall": 0.22539218166685782, "rouge1_recall_stderr": 0.002822899330851212, "rouge2_fmeasure": 0.025570973012891195, "rouge2_fmeasure_stderr": 0.0007692752377271742, "rouge2_precision": 0.02181756499211645, "rouge2_precision_stderr": 0.0007137545292173706, "rouge2_recall": 0.039804717049478786, "rouge2_recall_stderr": 0.0013560991274283968, "rougeL_fmeasure": 0.11505946082280019, "rougeL_fmeasure_stderr": 0.0012278934893675208, "rougeL_precision": 0.10068399069926871, "rougeL_precision_stderr": 0.0012243521909029603, "rougeL_recall": 0.17123882520643308, "rougeL_recall_stderr": 0.002168643066220908, "rougeLsum_fmeasure": 0.14484893653987274, "rougeLsum_fmeasure_stderr": 0.0016765021056686803, "rougeLsum_precision": 0.12780542015362023, "rougeLsum_precision_stderr": 0.0016861150014377962, "rougeLsum_recall": 0.21034780895799102, "rougeLsum_recall_stderr": 0.002605948309335639}, "rephrase_en": {"bleu": 0.4064700344068096, "bleu_stderr": 0.041071462910743026, "rouge1_fmeasure": 0.10124369006753632, "rouge1_fmeasure_stderr": 0.0012431775944759696, "rouge1_precision": 0.11008815005619565, "rouge1_precision_stderr": 0.0015367268621741792, "rouge1_recall": 0.12364976087148276, "rouge1_recall_stderr": 0.0017477601358293203, "rouge2_fmeasure": 0.00600392083872442, "rouge2_fmeasure_stderr": 0.00031876702476632513, "rouge2_precision": 0.006329778123956924, "rouge2_precision_stderr": 0.0003579493269657872, "rouge2_recall": 0.007946063514698743, "rouge2_recall_stderr": 0.00048230591130947426, "rougeL_fmeasure": 0.08055079492734035, "rougeL_fmeasure_stderr": 0.0009086151054529219, "rougeL_precision": 0.08810707825607618, "rougeL_precision_stderr": 0.0011970905402063467, "rougeL_recall": 0.0995681437810585, "rougeL_recall_stderr": 0.001372354219556691, "rougeLsum_fmeasure": 0.09678091628110796, "rougeLsum_fmeasure_stderr": 0.001168106008335189, "rougeLsum_precision": 0.10550757799818274, "rougeLsum_precision_stderr": 0.0014666803119605407, "rougeLsum_recall": 0.11816997810521175, "rougeLsum_recall_stderr": 0.0016494951956388857}, "summarize_above_en": {"bleu": 0.382732965383335, "bleu_stderr": 0.041457097975708616, "rouge1_fmeasure": 0.10049145155089874, "rouge1_fmeasure_stderr": 0.0011946171315062168, "rouge1_precision": 0.11362728657731791, "rouge1_precision_stderr": 0.0015714871657739951, "rouge1_recall": 0.11607779218894242, "rouge1_recall_stderr": 0.001575602946003843, "rouge2_fmeasure": 0.005194947294074261, "rouge2_fmeasure_stderr": 0.00030474043998879574, "rouge2_precision": 0.005836029460495955, "rouge2_precision_stderr": 0.00036095981618922905, "rouge2_recall": 0.0062244104824258515, "rouge2_recall_stderr": 0.0004249628756393917, "rougeL_fmeasure": 0.08052988976365942, "rougeL_fmeasure_stderr": 0.0008954855289212289, "rougeL_precision": 0.09152414632340847, "rougeL_precision_stderr": 0.0012419260737023521, "rougeL_recall": 0.09395918617125179, "rougeL_recall_stderr": 0.0012571326025089399, "rougeLsum_fmeasure": 0.09676724655543643, "rougeLsum_fmeasure_stderr": 0.0011401063414987313, "rougeLsum_precision": 0.10951406198494446, "rougeLsum_precision_stderr": 0.001514165066763457, "rougeLsum_recall": 0.11185509288980913, "rougeLsum_recall_stderr": 0.001509586047459457}, "tldr_en": {"bleu": 0.7735409292280904, "bleu_stderr": 0.04682416335531492, "rouge1_fmeasure": 0.11151220891599137, "rouge1_fmeasure_stderr": 0.0014668379581459778, "rouge1_precision": 0.11925916564183478, "rouge1_precision_stderr": 0.0017880379896240603, "rouge1_recall": 0.13943695708117368, "rouge1_recall_stderr": 0.002116433411707032, "rouge2_fmeasure": 0.010504603460980755, "rouge2_fmeasure_stderr": 0.0004959353821993032, "rouge2_precision": 0.010505863803598392, "rouge2_precision_stderr": 0.0005738646736412342, "rouge2_recall": 0.014684578264016469, "rouge2_recall_stderr": 0.0007964172087895143, "rougeL_fmeasure": 0.08956417926961593, "rougeL_fmeasure_stderr": 0.0011103097178602006, "rougeL_precision": 0.09588263877954578, "rougeL_precision_stderr": 0.0013996502096768437, "rougeL_recall": 0.11375527509576841, "rougeL_recall_stderr": 0.001731363662015009, "rougeLsum_fmeasure": 0.10583744233386468, "rougeLsum_fmeasure_stderr": 0.0013667666533331855, "rougeLsum_precision": 0.11339532279740656, "rougeLsum_precision_stderr": 0.0016872131131246602, "rougeLsum_recall": 0.1323364741919991, "rougeLsum_recall_stderr": 0.0019843253951356537}, "write_abstract_en": {"bleu": 0.3820141559457903, "bleu_stderr": 0.03590316359310684, "rouge1_fmeasure": 0.1016546124922999, "rouge1_fmeasure_stderr": 0.0012651922442427001, "rouge1_precision": 0.11013619429611844, "rouge1_precision_stderr": 0.001546664571183218, "rouge1_recall": 0.1232803535134769, "rouge1_recall_stderr": 0.0017517550316112635, "rouge2_fmeasure": 0.006048464517830181, "rouge2_fmeasure_stderr": 0.00032306729820669704, "rouge2_precision": 0.006234351643431517, "rouge2_precision_stderr": 0.00034125509784363245, "rouge2_recall": 0.007962049327295925, "rouge2_recall_stderr": 0.0005161503037287333, "rougeL_fmeasure": 0.08023398785251074, "rougeL_fmeasure_stderr": 0.0009106005118733982, "rougeL_precision": 0.0875404512463235, "rougeL_precision_stderr": 0.00118479594104557, "rougeL_recall": 0.0982995279262905, "rougeL_recall_stderr": 0.0013504657871243205, "rougeLsum_fmeasure": 0.09722400124042876, "rougeLsum_fmeasure_stderr": 0.0011899218963608768, "rougeLsum_precision": 0.10557342164225539, "rougeLsum_precision_stderr": 0.0014719765283516568, "rougeLsum_recall": 0.1177632536098584, "rougeLsum_recall_stderr": 0.0016421313291817262}}, "3": {"article_summary_en": {"bleu": 1.2271649835407419, "bleu_stderr": 0.061326948665848736, "rouge1_fmeasure": 0.12309846747867732, "rouge1_fmeasure_stderr": 0.0019272106431180854, "rouge1_precision": 0.11331660679440218, "rouge1_precision_stderr": 0.0020148980956489313, "rouge1_recall": 0.17682573132355542, "rouge1_recall_stderr": 0.0029418843134879813, "rouge2_fmeasure": 0.02030504236701163, "rouge2_fmeasure_stderr": 0.0007026247733576691, "rouge2_precision": 0.018033312403096684, "rouge2_precision_stderr": 0.0006783067936208244, "rouge2_recall": 0.031206199578028934, "rouge2_recall_stderr": 0.001235176208046927, "rougeL_fmeasure": 0.09410057985901603, "rougeL_fmeasure_stderr": 0.0013756456156665128, "rougeL_precision": 0.0867545256195891, "rougeL_precision_stderr": 0.0015118580414631715, "rougeL_recall": 0.13772453378421665, "rougeL_recall_stderr": 0.002277790929230465, "rougeLsum_fmeasure": 0.1147938850890483, "rougeLsum_fmeasure_stderr": 0.0017791199568731235, "rougeLsum_precision": 0.10596910703751164, "rougeLsum_precision_stderr": 0.001891907064541939, "rougeLsum_recall": 0.16492851491383875, "rougeLsum_recall_stderr": 0.0027267448285863516}, "rephrase_en": {"bleu": 0.5087314599386397, "bleu_stderr": 0.061212421752417735, "rouge1_fmeasure": 0.08498062296250532, "rouge1_fmeasure_stderr": 0.0014212404882767398, "rouge1_precision": 0.094117032806795, "rouge1_precision_stderr": 0.0017305123351559388, "rouge1_recall": 0.1061886960885055, "rouge1_recall_stderr": 0.0020438338074463868, "rouge2_fmeasure": 0.006494830300354191, "rouge2_fmeasure_stderr": 0.0003529124882286109, "rouge2_precision": 0.006623081561306862, "rouge2_precision_stderr": 0.0003792072547215317, "rouge2_recall": 0.00957912154003632, "rouge2_recall_stderr": 0.0006732767370116441, "rougeL_fmeasure": 0.0677917896388117, "rougeL_fmeasure_stderr": 0.001061516966060374, "rougeL_precision": 0.0760422274749195, "rougeL_precision_stderr": 0.0013874920699279642, "rougeL_recall": 0.08570210788487924, "rougeL_recall_stderr": 0.0016287020168935214, "rougeLsum_fmeasure": 0.08096977315853292, "rougeLsum_fmeasure_stderr": 0.0013335904643873365, "rougeLsum_precision": 0.08992646673675982, "rougeLsum_precision_stderr": 0.0016454052932894198, "rougeLsum_recall": 0.10125000826231054, "rougeLsum_recall_stderr": 0.001933208130218866}, "summarize_above_en": {"bleu": 0.3919914059035021, "bleu_stderr": 0.06209565085248539, "rouge1_fmeasure": 0.08043785817220254, "rouge1_fmeasure_stderr": 0.0013311625960759211, "rouge1_precision": 0.0955940850522753, "rouge1_precision_stderr": 0.0018130428010804185, "rouge1_recall": 0.09383021551741794, "rouge1_recall_stderr": 0.0017574878214917625, "rouge2_fmeasure": 0.00471973086085565, "rouge2_fmeasure_stderr": 0.00031285101854069914, "rouge2_precision": 0.005894324763379858, "rouge2_precision_stderr": 0.0005451846101275158, "rouge2_recall": 0.006111381952443814, "rouge2_recall_stderr": 0.0004988935962431043, "rougeL_fmeasure": 0.06517286449766353, "rougeL_fmeasure_stderr": 0.0010214075492220697, "rougeL_precision": 0.07845129854611516, "rougeL_precision_stderr": 0.001504183246143632, "rougeL_recall": 0.07671122424431512, "rougeL_recall_stderr": 0.0014337379247573415, "rougeLsum_fmeasure": 0.07705416712954397, "rougeLsum_fmeasure_stderr": 0.0012575955853530694, "rougeLsum_precision": 0.09173213439305535, "rougeLsum_precision_stderr": 0.0017274967484957875, "rougeLsum_recall": 0.08992169460486722, "rougeLsum_recall_stderr": 0.0016733690010191946}, "tldr_en": {"bleu": 0.8642314685339636, "bleu_stderr": 0.07105912584286404, "rouge1_fmeasure": 0.09780113881018691, "rouge1_fmeasure_stderr": 0.0016273262648512452, "rouge1_precision": 0.10733987691391439, "rouge1_precision_stderr": 0.0020889839913980418, "rouge1_recall": 0.12476072254733013, "rouge1_recall_stderr": 0.002308247813491843, "rouge2_fmeasure": 0.01096682747964428, "rouge2_fmeasure_stderr": 0.0005407448750885603, "rouge2_precision": 0.012191906060064263, "rouge2_precision_stderr": 0.0007006501461727509, "rouge2_recall": 0.01532295103209109, "rouge2_recall_stderr": 0.0008648292695711531, "rougeL_fmeasure": 0.08074375845019817, "rougeL_fmeasure_stderr": 0.0012686172017943162, "rougeL_precision": 0.08876565747817264, "rougeL_precision_stderr": 0.0017029372366386352, "rougeL_recall": 0.10463111270816423, "rougeL_recall_stderr": 0.0019111151657166508, "rougeLsum_fmeasure": 0.09202413237033855, "rougeLsum_fmeasure_stderr": 0.0015127002889302342, "rougeLsum_precision": 0.10123055336456097, "rougeLsum_precision_stderr": 0.0019729409528934246, "rougeLsum_recall": 0.11748223446367839, "rougeLsum_recall_stderr": 0.0021566918011451134}, "write_abstract_en": {"bleu": 0.5176377216004082, "bleu_stderr": 0.06866328234036077, "rouge1_fmeasure": 0.0785984819107557, "rouge1_fmeasure_stderr": 0.0014372932969101489, "rouge1_precision": 0.08899396863415189, "rouge1_precision_stderr": 0.001777790692308317, "rouge1_recall": 0.09692319541629159, "rouge1_recall_stderr": 0.0019946027284699383, "rouge2_fmeasure": 0.0058753493744521336, "rouge2_fmeasure_stderr": 0.00036684689106018846, "rouge2_precision": 0.005984354455890461, "rouge2_precision_stderr": 0.00040684067708898005, "rouge2_recall": 0.008510439649050199, "rouge2_recall_stderr": 0.0006316823704492148, "rougeL_fmeasure": 0.06287166017244038, "rougeL_fmeasure_stderr": 0.0010782265186488856, "rougeL_precision": 0.07217146798975321, "rougeL_precision_stderr": 0.0014091242615705865, "rougeL_recall": 0.07831148959974298, "rougeL_recall_stderr": 0.0015814306656436506, "rougeLsum_fmeasure": 0.07485251192796798, "rougeLsum_fmeasure_stderr": 0.0013431259530301976, "rougeLsum_precision": 0.08495497287249752, "rougeLsum_precision_stderr": 0.0016708456442186747, "rougeLsum_recall": 0.09237830202855782, "rougeLsum_recall_stderr": 0.0018778739354038604}}, "4": {"article_summary_en": {"bleu": 0.21901783251929063, "bleu_stderr": 0.023129015503409796, "rouge1_fmeasure": 0.039273927677435934, "rouge1_fmeasure_stderr": 0.0015037043115189227, "rouge1_precision": 0.038476995951992485, "rouge1_precision_stderr": 0.0016453155252656947, "rouge1_recall": 0.057681790969237874, "rouge1_recall_stderr": 0.0023058359041284843, "rouge2_fmeasure": 0.006731993501111477, "rouge2_fmeasure_stderr": 0.00045966773056801204, "rouge2_precision": 0.006055290873763252, "rouge2_precision_stderr": 0.0004556065968758473, "rouge2_recall": 0.011582702540716703, "rouge2_recall_stderr": 0.0009372131400483686, "rougeL_fmeasure": 0.030451500169627634, "rougeL_fmeasure_stderr": 0.0011231733238070457, "rougeL_precision": 0.0299699971593133, "rougeL_precision_stderr": 0.0012852804923565745, "rougeL_recall": 0.04584513362616462, "rougeL_recall_stderr": 0.0018409273324060942, "rougeLsum_fmeasure": 0.036456484857113516, "rougeLsum_fmeasure_stderr": 0.0013830180281845924, "rougeLsum_precision": 0.03588852826032929, "rougeLsum_precision_stderr": 0.0015400770868070532, "rougeLsum_recall": 0.05364153027915661, "rougeLsum_recall_stderr": 0.002137335190836827}, "rephrase_en": {"bleu": 0.05513061350682931, "bleu_stderr": 0.01363489004556789, "rouge1_fmeasure": 0.027959665353093514, "rouge1_fmeasure_stderr": 0.0011329907447503772, "rouge1_precision": 0.033017953252287624, "rouge1_precision_stderr": 0.001455074891420513, "rouge1_recall": 0.035407839416534886, "rouge1_recall_stderr": 0.0015340457724425114, "rouge2_fmeasure": 0.002417994702779381, "rouge2_fmeasure_stderr": 0.0002673599459103152, "rouge2_precision": 0.0028089869136647076, "rouge2_precision_stderr": 0.00035883040231905175, "rouge2_recall": 0.0033990005640009368, "rouge2_recall_stderr": 0.00043264989107393107, "rougeL_fmeasure": 0.022633019178426736, "rougeL_fmeasure_stderr": 0.0008874765902663727, "rougeL_precision": 0.027079006620999064, "rougeL_precision_stderr": 0.0012047423398949, "rougeL_recall": 0.02913572117563057, "rougeL_recall_stderr": 0.0012482614882508539, "rougeLsum_fmeasure": 0.02640979987744244, "rougeLsum_fmeasure_stderr": 0.0010615456146286408, "rougeLsum_precision": 0.031138458647991887, "rougeLsum_precision_stderr": 0.0013669267038162584, "rougeLsum_recall": 0.03352557536851953, "rougeLsum_recall_stderr": 0.0014450564773056973}, "summarize_above_en": {"bleu": 0.01607270818304334, "bleu_stderr": 0.0034624809654178012, "rouge1_fmeasure": 0.024625870173260112, "rouge1_fmeasure_stderr": 0.0010164821198364775, "rouge1_precision": 0.031069492435776818, "rouge1_precision_stderr": 0.0013777037826483293, "rouge1_recall": 0.02923924720251318, "rouge1_recall_stderr": 0.001332321685582368, "rouge2_fmeasure": 0.0018715335775846497, "rouge2_fmeasure_stderr": 0.0002104568234283972, "rouge2_precision": 0.0022741707476197034, "rouge2_precision_stderr": 0.0002784957287667364, "rouge2_recall": 0.002541556245852753, "rouge2_recall_stderr": 0.00034331230832384933, "rougeL_fmeasure": 0.020411284941508113, "rougeL_fmeasure_stderr": 0.0008173009244281204, "rougeL_precision": 0.026118959440308825, "rougeL_precision_stderr": 0.0011609305606255281, "rougeL_recall": 0.024492403731580736, "rougeL_recall_stderr": 0.0011008059304958085, "rougeLsum_fmeasure": 0.02329804778849391, "rougeLsum_fmeasure_stderr": 0.0009534761646565726, "rougeLsum_precision": 0.0295178758499726, "rougeLsum_precision_stderr": 0.0013119054474959343, "rougeLsum_recall": 0.027611772675785972, "rougeLsum_recall_stderr": 0.0012447336136434688}, "tldr_en": {"bleu": 0.11591220265958264, "bleu_stderr": 0.018810551681001538, "rouge1_fmeasure": 0.03531682035958441, "rouge1_fmeasure_stderr": 0.0013697458060726072, "rouge1_precision": 0.04130438911538399, "rouge1_precision_stderr": 0.0018399622962315205, "rouge1_recall": 0.04523262499737334, "rouge1_recall_stderr": 0.0018427581724112859, "rouge2_fmeasure": 0.005041922188233573, "rouge2_fmeasure_stderr": 0.00039035553600462724, "rouge2_precision": 0.0069981887164715815, "rouge2_precision_stderr": 0.0008066183569971981, "rouge2_recall": 0.00693668855148366, "rouge2_recall_stderr": 0.0006076952741663856, "rougeL_fmeasure": 0.029228194708275798, "rougeL_fmeasure_stderr": 0.001099845646230128, "rougeL_precision": 0.03459558732097317, "rougeL_precision_stderr": 0.0015673695864144438, "rougeL_recall": 0.037976749671683144, "rougeL_recall_stderr": 0.0015299014996893944, "rougeLsum_fmeasure": 0.032707886197975086, "rougeLsum_fmeasure_stderr": 0.001248884867759778, "rougeLsum_precision": 0.038549446328862175, "rougeLsum_precision_stderr": 0.0017184540964501843, "rougeLsum_recall": 0.041863381072326546, "rougeLsum_recall_stderr": 0.0016845634789296187}, "write_abstract_en": {"bleu": 0.013454379756596573, "bleu_stderr": 0.004377302392144288, "rouge1_fmeasure": 0.021150352711754538, "rouge1_fmeasure_stderr": 0.0010009476386843301, "rouge1_precision": 0.024653052383932526, "rouge1_precision_stderr": 0.0012369651915557867, "rouge1_recall": 0.026978552912746406, "rouge1_recall_stderr": 0.001367418485604499, "rouge2_fmeasure": 0.0018364476816945278, "rouge2_fmeasure_stderr": 0.00021456601406933008, "rouge2_precision": 0.0020847909583644186, "rouge2_precision_stderr": 0.00028781597725583064, "rouge2_recall": 0.0025592724240508426, "rouge2_recall_stderr": 0.00032606918523288823, "rougeL_fmeasure": 0.01695587958257084, "rougeL_fmeasure_stderr": 0.0007821077415757568, "rougeL_precision": 0.019887895435798185, "rougeL_precision_stderr": 0.0009981749275954795, "rougeL_recall": 0.02207509587363469, "rougeL_recall_stderr": 0.0011126244037750546, "rougeLsum_fmeasure": 0.019854394029124416, "rougeLsum_fmeasure_stderr": 0.0009334487521584576, "rougeLsum_precision": 0.023148124955408137, "rougeLsum_precision_stderr": 0.001155778717437976, "rougeLsum_recall": 0.025400258275254814, "rougeLsum_recall_stderr": 0.0012784306268269182}}, "5": {"article_summary_en": {"bleu": 5.322880476060286e-08, "bleu_stderr": 1.5191889091292856e-07, "rouge1_fmeasure": 0.006323793453713114, "rouge1_fmeasure_stderr": 0.000679766707607541, "rouge1_precision": 0.00741304726602641, "rouge1_precision_stderr": 0.0009755620107809973, "rouge1_recall": 0.009480629169855271, "rouge1_recall_stderr": 0.0010572884760238043, "rouge2_fmeasure": 0.0012665122562968891, "rouge2_fmeasure_stderr": 0.00021596794467997575, "rouge2_precision": 0.001214496073168751, "rouge2_precision_stderr": 0.00022011821368556766, "rouge2_recall": 0.0021696621911090567, "rouge2_recall_stderr": 0.00045950435492140597, "rougeL_fmeasure": 0.004971201029477744, "rougeL_fmeasure_stderr": 0.0005078524339863577, "rougeL_precision": 0.006028681674848774, "rougeL_precision_stderr": 0.0008454472955138656, "rougeL_recall": 0.007771898833374424, "rougeL_recall_stderr": 0.0008802017587782391, "rougeLsum_fmeasure": 0.005929594045281956, "rougeLsum_fmeasure_stderr": 0.0006291046983741132, "rougeLsum_precision": 0.007036952939226733, "rougeLsum_precision_stderr": 0.0009413610021170078, "rougeLsum_recall": 0.008965157586393747, "rougeLsum_recall_stderr": 0.0009950047096009401}, "rephrase_en": {"bleu": 1.9107654017640184e-11, "bleu_stderr": 1.3525115607349213e-09, "rouge1_fmeasure": 0.0042516054308208645, "rouge1_fmeasure_stderr": 0.00047637308191102623, "rouge1_precision": 0.004907621925869614, "rouge1_precision_stderr": 0.0005763406359517721, "rouge1_recall": 0.0053768195418473184, "rouge1_recall_stderr": 0.0006637161154816707, "rouge2_fmeasure": 0.00036529417385801794, "rouge2_fmeasure_stderr": 9.087182991829831e-05, "rouge2_precision": 0.00047314584408024164, "rouge2_precision_stderr": 0.00011906110853937211, "rouge2_recall": 0.0004964345162201845, "rouge2_recall_stderr": 0.00016880696249774884, "rougeL_fmeasure": 0.00343786505841611, "rougeL_fmeasure_stderr": 0.0003771726212323476, "rougeL_precision": 0.0040961836397586295, "rougeL_precision_stderr": 0.00048820210285777777, "rougeL_recall": 0.004329727791926256, "rougeL_recall_stderr": 0.0005262040146135757, "rougeLsum_fmeasure": 0.004022188513290089, "rougeLsum_fmeasure_stderr": 0.00044739410245404963, "rougeLsum_precision": 0.004702148837329425, "rougeLsum_precision_stderr": 0.000555612812542979, "rougeLsum_recall": 0.005058029292746475, "rougeLsum_recall_stderr": 0.0006171534300008613}, "summarize_above_en": {"bleu": 4.950641247853647e-16, "bleu_stderr": 3.846396789441964e-14, "rouge1_fmeasure": 0.003521129990818492, "rouge1_fmeasure_stderr": 0.0004036686379505131, "rouge1_precision": 0.0046970714312177565, "rouge1_precision_stderr": 0.0005792165335246764, "rouge1_recall": 0.004114523129900782, "rouge1_recall_stderr": 0.0005483319592202135, "rouge2_fmeasure": 0.00020048952920264672, "rouge2_fmeasure_stderr": 6.865942316355645e-05, "rouge2_precision": 0.0002427708680341301, "rouge2_precision_stderr": 7.969113434382603e-05, "rouge2_recall": 0.00038997340658384743, "rouge2_recall_stderr": 0.00018348649094998182, "rougeL_fmeasure": 0.00304301527398751, "rougeL_fmeasure_stderr": 0.00034135496557363107, "rougeL_precision": 0.0041148184629156, "rougeL_precision_stderr": 0.0005057838120170794, "rougeL_recall": 0.003574300499782594, "rougeL_recall_stderr": 0.00047502389608598274, "rougeLsum_fmeasure": 0.0033595582196200113, "rougeLsum_fmeasure_stderr": 0.0003831160597743695, "rougeLsum_precision": 0.004523141975654186, "rougeLsum_precision_stderr": 0.0005607431680479185, "rougeLsum_recall": 0.00392102854568574, "rougeLsum_recall_stderr": 0.0005176638248032069}, "tldr_en": {"bleu": 6.457241109219731e-10, "bleu_stderr": 4.3951639243907024e-09, "rouge1_fmeasure": 0.00646829859544361, "rouge1_fmeasure_stderr": 0.0006749836942920999, "rouge1_precision": 0.00840179041155278, "rouge1_precision_stderr": 0.0009319251831963879, "rouge1_recall": 0.007933117544594833, "rouge1_recall_stderr": 0.0008986804343115746, "rouge2_fmeasure": 0.0010361138600270753, "rouge2_fmeasure_stderr": 0.00019163061331141026, "rouge2_precision": 0.001300602211987841, "rouge2_precision_stderr": 0.0002505531879453807, "rouge2_recall": 0.0015037014195862734, "rouge2_recall_stderr": 0.00033970791079948175, "rougeL_fmeasure": 0.0052628072217658275, "rougeL_fmeasure_stderr": 0.0005418137104031803, "rougeL_precision": 0.0069081946455306804, "rougeL_precision_stderr": 0.000767045855376856, "rougeL_recall": 0.006517809586183419, "rougeL_recall_stderr": 0.0007338789355087525, "rougeLsum_fmeasure": 0.006026061380800861, "rougeLsum_fmeasure_stderr": 0.0006299887360163144, "rougeLsum_precision": 0.007887486215836917, "rougeLsum_precision_stderr": 0.0008833842095846506, "rougeLsum_recall": 0.007316433171910116, "rougeLsum_recall_stderr": 0.0008198190943819032}, "write_abstract_en": {"bleu": 7.304724850661707e-22, "bleu_stderr": 4.651649551405445e-19, "rouge1_fmeasure": 0.0023404225914027293, "rouge1_fmeasure_stderr": 0.00035897358936479027, "rouge1_precision": 0.002594820378060942, "rouge1_precision_stderr": 0.00041260070427087084, "rouge1_recall": 0.00315946115980561, "rouge1_recall_stderr": 0.0005463787261087583, "rouge2_fmeasure": 0.0002340501971901335, "rouge2_fmeasure_stderr": 7.27640076429073e-05, "rouge2_precision": 0.00020295754731072598, "rouge2_precision_stderr": 6.30448872924687e-05, "rouge2_recall": 0.0004422474374560793, "rouge2_recall_stderr": 0.00017450355164885773, "rougeL_fmeasure": 0.0018292868173016377, "rougeL_fmeasure_stderr": 0.0002697379131437824, "rougeL_precision": 0.002059754228694586, "rougeL_precision_stderr": 0.0003240033862130502, "rougeL_recall": 0.002522670751072346, "rougeL_recall_stderr": 0.00043340282734999423, "rougeLsum_fmeasure": 0.0022043625397931894, "rougeLsum_fmeasure_stderr": 0.00033340306926608464, "rougeLsum_precision": 0.002476388701828487, "rougeLsum_precision_stderr": 0.0003949847874533559, "rougeLsum_recall": 0.0029610416476689777, "rougeLsum_recall_stderr": 0.0005018145536321888}}}, "anli_r1": {"0": {"GPT-3 style": {"acc": 0.323, "acc_norm": 0.333, "acc_norm_stderr": 0.014910846164229857, "acc_stderr": 0.014794927843348633, "subset": 1}, "MNLI crowdsource": {"acc": 0.321, "acc_norm": 0.334, "acc_norm_stderr": 0.014922019523732967, "acc_stderr": 0.014770821817934654, "subset": 1}, "can we infer": {"acc": 0.321, "acc_norm": 0.337, "acc_norm_stderr": 0.014955087918653603, "acc_stderr": 0.01477082181793465, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.318, "acc_norm": 0.333, "acc_norm_stderr": 0.014910846164229857, "acc_stderr": 0.014734079309311901, "subset": 1}, "justified in saying": {"acc": 0.32, "acc_norm": 0.337, "acc_norm_stderr": 0.014955087918653596, "acc_stderr": 0.014758652303574883, "subset": 1}}, "1": {"GPT-3 style": {"acc": 0.33, "acc_norm": 0.326, "acc_norm_stderr": 0.014830507204541037, "acc_stderr": 0.014876872027456732, "subset": 1}, "MNLI crowdsource": {"acc": 0.333, "acc_norm": 0.333, "acc_norm_stderr": 0.014910846164229863, "acc_stderr": 0.014910846164229863, "subset": 1}, "can we infer": {"acc": 0.33, "acc_norm": 0.333, "acc_norm_stderr": 0.014910846164229863, "acc_stderr": 0.014876872027456732, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.333, "acc_norm": 0.333, "acc_norm_stderr": 0.014910846164229863, "acc_stderr": 0.014910846164229863, "subset": 1}, "justified in saying": {"acc": 0.327, "acc_norm": 0.328, "acc_norm_stderr": 0.014853842487270336, "acc_stderr": 0.014842213153411242, "subset": 1}}, "2": {"GPT-3 style": {"acc": 0.354, "acc_norm": 0.35, "acc_norm_stderr": 0.015090650341444236, "acc_stderr": 0.015129868238451772, "subset": 1}, "MNLI crowdsource": {"acc": 0.338, "acc_norm": 0.336, "acc_norm_stderr": 0.014944140233795021, "acc_stderr": 0.014965960710224485, "subset": 1}, "can we infer": {"acc": 0.342, "acc_norm": 0.343, "acc_norm_stderr": 0.015019206922356953, "acc_stderr": 0.01500870618212173, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.336, "acc_norm": 0.337, "acc_norm_stderr": 0.014955087918653603, "acc_stderr": 0.014944140233795021, "subset": 1}, "justified in saying": {"acc": 0.346, "acc_norm": 0.353, "acc_norm_stderr": 0.015120172605483706, "acc_stderr": 0.015050266127564431, "subset": 1}}, "3": {"GPT-3 style": {"acc": 0.347, "acc_norm": 0.346, "acc_norm_stderr": 0.015050266127564436, "acc_stderr": 0.015060472031706618, "subset": 1}, "MNLI crowdsource": {"acc": 0.373, "acc_norm": 0.361, "acc_norm_stderr": 0.015195720118175124, "acc_stderr": 0.015300493622922814, "subset": 1}, "can we infer": {"acc": 0.349, "acc_norm": 0.36, "acc_norm_stderr": 0.015186527932040115, "acc_stderr": 0.0150806639915631, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.339, "acc_norm": 0.338, "acc_norm_stderr": 0.01496596071022448, "acc_stderr": 0.014976758771620342, "subset": 1}, "justified in saying": {"acc": 0.351, "acc_norm": 0.35, "acc_norm_stderr": 0.015090650341444236, "acc_stderr": 0.015100563798316403, "subset": 1}}, "4": {"GPT-3 style": {"acc": 0.337, "acc_norm": 0.345, "acc_norm_stderr": 0.015039986742055235, "acc_stderr": 0.014955087918653593, "subset": 1}, "MNLI crowdsource": {"acc": 0.346, "acc_norm": 0.342, "acc_norm_stderr": 0.015008706182121728, "acc_stderr": 0.015050266127564446, "subset": 1}, "can we infer": {"acc": 0.341, "acc_norm": 0.337, "acc_norm_stderr": 0.014955087918653602, "acc_stderr": 0.014998131348402709, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.319, "acc_norm": 0.326, "acc_norm_stderr": 0.014830507204541044, "acc_stderr": 0.014746404865473486, "subset": 1}, "justified in saying": {"acc": 0.342, "acc_norm": 0.347, "acc_norm_stderr": 0.015060472031706618, "acc_stderr": 0.015008706182121726, "subset": 1}}, "5": {"GPT-3 style": {"acc": 0.344, "acc_norm": 0.333, "acc_norm_stderr": 0.01491084616422987, "acc_stderr": 0.015029633724408948, "subset": 1}, "MNLI crowdsource": {"acc": 0.338, "acc_norm": 0.329, "acc_norm_stderr": 0.01486539538592836, "acc_stderr": 0.014965960710224485, "subset": 1}, "can we infer": {"acc": 0.332, "acc_norm": 0.33, "acc_norm_stderr": 0.014876872027456736, "acc_stderr": 0.01489959724281149, "subset": 1}, "guaranteed/possible/impossible": {"acc": 0.324, "acc_norm": 0.324, "acc_norm_stderr": 0.01480686473373886, "acc_stderr": 0.01480686473373886, "subset": 1}, "justified in saying": {"acc": 0.339, "acc_norm": 0.321, "acc_norm_stderr": 0.014770821817934652, "acc_stderr": 0.014976758771620337, "subset": 1}}}, "anli_r2": {"0": {"GPT-3 style": {"acc": 0.32, "acc_norm": 0.333, "acc_norm_stderr": 0.01491084616422987, "acc_stderr": 0.014758652303574888, "subset": 2}, "MNLI crowdsource": {"acc": 0.336, "acc_norm": 0.344, "acc_norm_stderr": 0.015029633724408941, "acc_stderr": 0.01494414023379502, "subset": 2}, "can we infer": {"acc": 0.336, "acc_norm": 0.332, "acc_norm_stderr": 0.014899597242811488, "acc_stderr": 0.01494414023379502, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.348, "acc_norm": 0.329, "acc_norm_stderr": 0.014865395385928352, "acc_stderr": 0.015070604603768408, "subset": 2}, "justified in saying": {"acc": 0.311, "acc_norm": 0.329, "acc_norm_stderr": 0.014865395385928367, "acc_stderr": 0.014645596385722694, "subset": 2}}, "1": {"GPT-3 style": {"acc": 0.318, "acc_norm": 0.315, "acc_norm_stderr": 0.014696631960792508, "acc_stderr": 0.014734079309311901, "subset": 2}, "MNLI crowdsource": {"acc": 0.315, "acc_norm": 0.315, "acc_norm_stderr": 0.014696631960792506, "acc_stderr": 0.014696631960792506, "subset": 2}, "can we infer": {"acc": 0.318, "acc_norm": 0.318, "acc_norm_stderr": 0.014734079309311901, "acc_stderr": 0.014734079309311901, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.315, "acc_norm": 0.315, "acc_norm_stderr": 0.014696631960792506, "acc_stderr": 0.014696631960792506, "subset": 2}, "justified in saying": {"acc": 0.32, "acc_norm": 0.319, "acc_norm_stderr": 0.014746404865473479, "acc_stderr": 0.014758652303574886, "subset": 2}}, "2": {"GPT-3 style": {"acc": 0.329, "acc_norm": 0.324, "acc_norm_stderr": 0.014806864733738864, "acc_stderr": 0.014865395385928373, "subset": 2}, "MNLI crowdsource": {"acc": 0.321, "acc_norm": 0.326, "acc_norm_stderr": 0.014830507204541033, "acc_stderr": 0.014770821817934647, "subset": 2}, "can we infer": {"acc": 0.324, "acc_norm": 0.321, "acc_norm_stderr": 0.014770821817934645, "acc_stderr": 0.01480686473373886, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.32, "acc_norm": 0.317, "acc_norm_stderr": 0.014721675438880213, "acc_stderr": 0.01475865230357489, "subset": 2}, "justified in saying": {"acc": 0.311, "acc_norm": 0.322, "acc_norm_stderr": 0.014782913600996685, "acc_stderr": 0.014645596385722692, "subset": 2}}, "3": {"GPT-3 style": {"acc": 0.325, "acc_norm": 0.317, "acc_norm_stderr": 0.014721675438880217, "acc_stderr": 0.014818724459095524, "subset": 2}, "MNLI crowdsource": {"acc": 0.325, "acc_norm": 0.329, "acc_norm_stderr": 0.014865395385928362, "acc_stderr": 0.014818724459095524, "subset": 2}, "can we infer": {"acc": 0.337, "acc_norm": 0.341, "acc_norm_stderr": 0.01499813134840271, "acc_stderr": 0.014955087918653596, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.326, "acc_norm": 0.328, "acc_norm_stderr": 0.014853842487270334, "acc_stderr": 0.014830507204541038, "subset": 2}, "justified in saying": {"acc": 0.333, "acc_norm": 0.341, "acc_norm_stderr": 0.01499813134840271, "acc_stderr": 0.014910846164229868, "subset": 2}}, "4": {"GPT-3 style": {"acc": 0.336, "acc_norm": 0.322, "acc_norm_stderr": 0.014782913600996681, "acc_stderr": 0.014944140233795021, "subset": 2}, "MNLI crowdsource": {"acc": 0.325, "acc_norm": 0.328, "acc_norm_stderr": 0.014853842487270334, "acc_stderr": 0.014818724459095524, "subset": 2}, "can we infer": {"acc": 0.328, "acc_norm": 0.336, "acc_norm_stderr": 0.01494414023379502, "acc_stderr": 0.01485384248727033, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.325, "acc_norm": 0.32, "acc_norm_stderr": 0.014758652303574883, "acc_stderr": 0.014818724459095524, "subset": 2}, "justified in saying": {"acc": 0.329, "acc_norm": 0.323, "acc_norm_stderr": 0.014794927843348633, "acc_stderr": 0.01486539538592837, "subset": 2}}, "5": {"GPT-3 style": {"acc": 0.333, "acc_norm": 0.33, "acc_norm_stderr": 0.01487687202745673, "acc_stderr": 0.014910846164229871, "subset": 2}, "MNLI crowdsource": {"acc": 0.303, "acc_norm": 0.317, "acc_norm_stderr": 0.014721675438880226, "acc_stderr": 0.01453968371053525, "subset": 2}, "can we infer": {"acc": 0.336, "acc_norm": 0.325, "acc_norm_stderr": 0.014818724459095524, "acc_stderr": 0.014944140233795021, "subset": 2}, "guaranteed/possible/impossible": {"acc": 0.317, "acc_norm": 0.314, "acc_norm_stderr": 0.014683991951087964, "acc_stderr": 0.014721675438880224, "subset": 2}, "justified in saying": {"acc": 0.328, "acc_norm": 0.337, "acc_norm_stderr": 0.014955087918653596, "acc_stderr": 0.014853842487270336, "subset": 2}}}, "anli_r3": {"0": {"GPT-3 style": {"acc": 0.30583333333333335, "acc_norm": 0.335, "acc_norm_stderr": 0.01363087184382147, "acc_stderr": 0.013306526255831147, "subset": 3}, "MNLI crowdsource": {"acc": 0.335, "acc_norm": 0.34, "acc_norm_stderr": 0.013680495725767787, "acc_stderr": 0.013630871843821477, "subset": 3}, "can we infer": {"acc": 0.3433333333333333, "acc_norm": 0.3441666666666667, "acc_norm_stderr": 0.013720551062295756, "acc_stderr": 0.01371263383046586, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.32166666666666666, "acc_norm": 0.32916666666666666, "acc_norm_stderr": 0.01357080625843362, "acc_stderr": 0.013490095282989521, "subset": 3}, "justified in saying": {"acc": 0.35, "acc_norm": 0.3308333333333333, "acc_norm_stderr": 0.013588208070709002, "acc_stderr": 0.013774667009018552, "subset": 3}}, "1": {"GPT-3 style": {"acc": 0.3408333333333333, "acc_norm": 0.3333333333333333, "acc_norm_stderr": 0.013613950010225608, "acc_stderr": 0.013688600793296932, "subset": 3}, "MNLI crowdsource": {"acc": 0.33666666666666667, "acc_norm": 0.33666666666666667, "acc_norm_stderr": 0.013647602942406393, "acc_stderr": 0.013647602942406393, "subset": 3}, "can we infer": {"acc": 0.3441666666666667, "acc_norm": 0.3425, "acc_norm_stderr": 0.013704669762934725, "acc_stderr": 0.013720551062295756, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.33666666666666667, "acc_norm": 0.33666666666666667, "acc_norm_stderr": 0.013647602942406393, "acc_stderr": 0.013647602942406393, "subset": 3}, "justified in saying": {"acc": 0.3466666666666667, "acc_norm": 0.33916666666666667, "acc_norm_stderr": 0.013672343491681819, "acc_stderr": 0.013744022550571949, "subset": 3}}, "2": {"GPT-3 style": {"acc": 0.3416666666666667, "acc_norm": 0.33416666666666667, "acc_norm_stderr": 0.013622434813136767, "acc_stderr": 0.013696658778002519, "subset": 3}, "MNLI crowdsource": {"acc": 0.3433333333333333, "acc_norm": 0.3283333333333333, "acc_norm_stderr": 0.013562032919529015, "acc_stderr": 0.01371263383046586, "subset": 3}, "can we infer": {"acc": 0.335, "acc_norm": 0.3283333333333333, "acc_norm_stderr": 0.013562032919529019, "acc_stderr": 0.01363087184382147, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.30916666666666665, "acc_norm": 0.30666666666666664, "acc_norm_stderr": 0.013316642319070695, "acc_stderr": 0.013346684134591941, "subset": 3}, "justified in saying": {"acc": 0.3275, "acc_norm": 0.33166666666666667, "acc_norm_stderr": 0.01359683672948516, "acc_stderr": 0.013553211167251953, "subset": 3}}, "3": {"GPT-3 style": {"acc": 0.35333333333333333, "acc_norm": 0.3575, "acc_norm_stderr": 0.013840921245257787, "acc_stderr": 0.01380457216231493, "subset": 3}, "MNLI crowdsource": {"acc": 0.325, "acc_norm": 0.32166666666666666, "acc_norm_stderr": 0.013490095282989521, "acc_stderr": 0.013526454480351023, "subset": 3}, "can we infer": {"acc": 0.3375, "acc_norm": 0.3425, "acc_norm_stderr": 0.013704669762934734, "acc_stderr": 0.013655897185463667, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.3125, "acc_norm": 0.3125, "acc_norm_stderr": 0.013386029277441229, "acc_stderr": 0.013386029277441229, "subset": 3}, "justified in saying": {"acc": 0.3458333333333333, "acc_norm": 0.3425, "acc_norm_stderr": 0.013704669762934734, "acc_stderr": 0.013736245342311012, "subset": 3}}, "4": {"GPT-3 style": {"acc": 0.3416666666666667, "acc_norm": 0.3308333333333333, "acc_norm_stderr": 0.013588208070708995, "acc_stderr": 0.013696658778002519, "subset": 3}, "MNLI crowdsource": {"acc": 0.31416666666666665, "acc_norm": 0.30833333333333335, "acc_norm_stderr": 0.013336721143136467, "acc_stderr": 0.013405399314984101, "subset": 3}, "can we infer": {"acc": 0.3358333333333333, "acc_norm": 0.32666666666666666, "acc_norm_stderr": 0.013544340907003665, "acc_stderr": 0.01363926119093287, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.31166666666666665, "acc_norm": 0.30583333333333335, "acc_norm_stderr": 0.013306526255831164, "acc_stderr": 0.013376268790982112, "subset": 3}, "justified in saying": {"acc": 0.33416666666666667, "acc_norm": 0.3308333333333333, "acc_norm_stderr": 0.013588208070708986, "acc_stderr": 0.013622434813136764, "subset": 3}}, "5": {"GPT-3 style": {"acc": 0.3233333333333333, "acc_norm": 0.315, "acc_norm_stderr": 0.013415009084004859, "acc_stderr": 0.013508372867300231, "subset": 3}, "MNLI crowdsource": {"acc": 0.30916666666666665, "acc_norm": 0.31166666666666665, "acc_norm_stderr": 0.013376268790982112, "acc_stderr": 0.013346684134591945, "subset": 3}, "can we infer": {"acc": 0.3258333333333333, "acc_norm": 0.33166666666666667, "acc_norm_stderr": 0.013596836729485163, "acc_stderr": 0.013535422043417471, "subset": 3}, "guaranteed/possible/impossible": {"acc": 0.32416666666666666, "acc_norm": 0.32083333333333336, "acc_norm_stderr": 0.013480882752851555, "acc_stderr": 0.013517438120881629, "subset": 3}, "justified in saying": {"acc": 0.3275, "acc_norm": 0.3275, "acc_norm_stderr": 0.01355321116725195, "acc_stderr": 0.01355321116725195, "subset": 3}}}, "arc_easy": {"0": {"heres_a_problem": {"acc": 0.23293515358361774, "acc_norm": 0.23293515358361774, "acc_norm_stderr": 0.012352507042617396, "acc_stderr": 0.012352507042617396}, "i_am_hesitating": {"acc": 0.20563139931740615, "acc_norm": 0.25426621160409557, "acc_norm_stderr": 0.012724999945157741, "acc_stderr": 0.011810745260742578}, "multiple_choice": {"acc": 0.2235494880546075, "acc_norm": 0.28071672354948807, "acc_norm_stderr": 0.013131238126975586, "acc_stderr": 0.01217489663120261}, "pick_the_most_correct_option": {"acc": 0.23464163822525597, "acc_norm": 0.23464163822525597, "acc_norm_stderr": 0.012383873560768675, "acc_stderr": 0.012383873560768675}, "qa_options": {"acc": 0.29208754208754206, "acc_norm": 0.2756734006734007, "acc_norm_stderr": 0.009169229476542569, "acc_stderr": 0.009330705616569072}}, "1": {"heres_a_problem": {"acc": 0.24061433447098976, "acc_norm": 0.24061433447098976, "acc_norm_stderr": 0.012491468532390573, "acc_stderr": 0.012491468532390573}, "i_am_hesitating": {"acc": 0.19795221843003413, "acc_norm": 0.26023890784982934, "acc_norm_stderr": 0.01282193022511256, "acc_stderr": 0.011643990971573401}, "multiple_choice": {"acc": 0.28535353535353536, "acc_norm": 0.2735690235690236, "acc_norm_stderr": 0.009147424438490741, "acc_stderr": 0.009266280584997753}, "pick_the_most_correct_option": {"acc": 0.24873737373737373, "acc_norm": 0.24873737373737373, "acc_norm_stderr": 0.008870224411653797, "acc_stderr": 0.008870224411653797}, "qa_options": {"acc": 0.2975589225589226, "acc_norm": 0.2807239057239057, "acc_norm_stderr": 0.009220526174711356, "acc_stderr": 0.009381226721815539}}, "2": {"heres_a_problem": {"acc": 0.2558922558922559, "acc_norm": 0.2558922558922559, "acc_norm_stderr": 0.008953950243013991, "acc_stderr": 0.008953950243013991}, "i_am_hesitating": {"acc": 0.29797979797979796, "acc_norm": 0.27104377104377103, "acc_norm_stderr": 0.0091209197417606, "acc_stderr": 0.00938504606669487}, "multiple_choice": {"acc": 0.28324915824915825, "acc_norm": 0.2786195286195286, "acc_norm_stderr": 0.009199329195026348, "acc_stderr": 0.009245632200075456}, "pick_the_most_correct_option": {"acc": 0.2474747474747475, "acc_norm": 0.2474747474747475, "acc_norm_stderr": 0.00885511441483471, "acc_stderr": 0.00885511441483471}, "qa_options": {"acc": 0.29545454545454547, "acc_norm": 0.2824074074074074, "acc_norm_stderr": 0.009237303403479327, "acc_stderr": 0.009361987126556458}}, "3": {"heres_a_problem": {"acc": 0.255050505050505, "acc_norm": 0.255050505050505, "acc_norm_stderr": 0.008944265906130709, "acc_stderr": 0.008944265906130709}, "i_am_hesitating": {"acc": 0.28703703703703703, "acc_norm": 0.273989898989899, "acc_norm_stderr": 0.009151805901544028, "acc_stderr": 0.009282621598983068}, "multiple_choice": {"acc": 0.29292929292929293, "acc_norm": 0.2857744107744108, "acc_norm_stderr": 0.009270380606981212, "acc_stderr": 0.009338583737393607}, "pick_the_most_correct_option": {"acc": 0.23293515358361774, "acc_norm": 0.23293515358361774, "acc_norm_stderr": 0.012352507042617393, "acc_stderr": 0.012352507042617393}, "qa_options": {"acc": 0.29124579124579125, "acc_norm": 0.2828282828282828, "acc_norm_stderr": 0.009241472775328231, "acc_stderr": 0.009322788837938852}}, "4": {"heres_a_problem": {"acc": 0.2363481228668942, "acc_norm": 0.2363481228668942, "acc_norm_stderr": 0.012414960524301842, "acc_stderr": 0.012414960524301842}, "i_am_hesitating": {"acc": 0.2030716723549488, "acc_norm": 0.2568259385665529, "acc_norm_stderr": 0.0127669237941168, "acc_stderr": 0.011755899303705583}, "multiple_choice": {"acc": 0.2946127946127946, "acc_norm": 0.2828282828282828, "acc_norm_stderr": 0.00924147277532823, "acc_stderr": 0.009354224395837087}, "pick_the_most_correct_option": {"acc": 0.24873737373737373, "acc_norm": 0.24873737373737373, "acc_norm_stderr": 0.008870224411653797, "acc_stderr": 0.008870224411653797}, "qa_options": {"acc": 0.3005050505050505, "acc_norm": 0.2828282828282828, "acc_norm_stderr": 0.009241472775328228, "acc_stderr": 0.009407763090599316}}, "5": {"heres_a_problem": {"acc": 0.23378839590443687, "acc_norm": 0.23378839590443687, "acc_norm_stderr": 0.012368225378507161, "acc_stderr": 0.012368225378507161}, "i_am_hesitating": {"acc": 0.19965870307167236, "acc_norm": 0.26023890784982934, "acc_norm_stderr": 0.012821930225112556, "acc_stderr": 0.011681625756888692}, "multiple_choice": {"acc": 0.23122866894197952, "acc_norm": 0.26109215017064846, "acc_norm_stderr": 0.012835523909473845, "acc_stderr": 0.01232085883477228}, "pick_the_most_correct_option": {"acc": 0.22866894197952217, "acc_norm": 0.22866894197952217, "acc_norm_stderr": 0.012272853582540813, "acc_stderr": 0.012272853582540813}, "qa_options": {"acc": 0.2098976109215017, "acc_norm": 0.26023890784982934, "acc_norm_stderr": 0.012821930225112552, "acc_stderr": 0.011900548748047454}}}, "boolq": {"0": {"GPT-3 Style": {"acc": 0.422, "acc_norm": 0.6236666666666667, "acc_norm_stderr": 0.008846558976258922, "acc_stderr": 0.009018450207660421}, "after_reading": {"acc": 0.6236666666666667, "acc_norm": 0.543, "acc_norm_stderr": 0.00909640486825282, "acc_stderr": 0.008846558976258922}, "exercise": {"acc": 0.6236666666666667, "acc_norm": 0.3983333333333333, "acc_norm_stderr": 0.008939496695192017, "acc_stderr": 0.008846558976258922}, "valid_binary": {"acc": 0.6226666666666667, "acc_norm": 0.441, "acc_norm_stderr": 0.009066443632063164, "acc_stderr": 0.008851200156534391}, "yes_no_question": {"acc": 0.6243333333333333, "acc_norm": 0.6236666666666667, "acc_norm_stderr": 0.008846558976258922, "acc_stderr": 0.008843442555522142}}, "1": {"GPT-3 Style": {"acc": 0.539, "acc_norm": 0.5456666666666666, "acc_norm_stderr": 0.009092070195065414, "acc_stderr": 0.009102414587191052}, "after_reading": {"acc": 0.5426666666666666, "acc_norm": 0.5406666666666666, "acc_norm_stderr": 0.009099982269204863, "acc_stderr": 0.009096928229880423}, "exercise": {"acc": 0.546, "acc_norm": 0.5436666666666666, "acc_norm_stderr": 0.009095345834327865, "acc_stderr": 0.009091509877386517}, "valid_binary": {"acc": 0.5406666666666666, "acc_norm": 0.541, "acc_norm_stderr": 0.009099483512819305, "acc_stderr": 0.009099982269204863}, "yes_no_question": {"acc": 0.547, "acc_norm": 0.552, "acc_norm_stderr": 0.00908072059340538, "acc_stderr": 0.00908980426158007}}, "2": {"GPT-3 Style": {"acc": 0.5253333333333333, "acc_norm": 0.578, "acc_norm_stderr": 0.009018450207660424, "acc_stderr": 0.00911850443412263}, "after_reading": {"acc": 0.5673333333333334, "acc_norm": 0.5543333333333333, "acc_norm_stderr": 0.009076164124491365, "acc_stderr": 0.00904706345689798}, "exercise": {"acc": 0.5633333333333334, "acc_norm": 0.5506666666666666, "acc_norm_stderr": 0.009083233528874796, "acc_stderr": 0.009056690207178121}, "valid_binary": {"acc": 0.545, "acc_norm": 0.526, "acc_norm_stderr": 0.009117878695396636, "acc_stderr": 0.009093178503605508}, "yes_no_question": {"acc": 0.591, "acc_norm": 0.5963333333333334, "acc_norm_stderr": 0.008959169522662578, "acc_stderr": 0.008977742846741013}}, "3": {"GPT-3 Style": {"acc": 0.5316666666666666, "acc_norm": 0.58, "acc_norm_stderr": 0.009012606487132152, "acc_stderr": 0.009111901539047272}, "after_reading": {"acc": 0.5646666666666667, "acc_norm": 0.5616666666666666, "acc_norm_stderr": 0.009060524549335622, "acc_stderr": 0.009053547904033172}, "exercise": {"acc": 0.5626666666666666, "acc_norm": 0.5496666666666666, "acc_norm_stderr": 0.009085074954912701, "acc_stderr": 0.009058236409215862}, "valid_binary": {"acc": 0.5583333333333333, "acc_norm": 0.546, "acc_norm_stderr": 0.009091509877386517, "acc_stderr": 0.009067881941319678}, "yes_no_question": {"acc": 0.5976666666666667, "acc_norm": 0.608, "acc_norm_stderr": 0.008914697075129006, "acc_stderr": 0.008954354670397114}}, "4": {"GPT-3 Style": {"acc": 0.5383333333333333, "acc_norm": 0.5776666666666667, "acc_norm_stderr": 0.009019409415904176, "acc_stderr": 0.009103358843448796}, "after_reading": {"acc": 0.5673333333333334, "acc_norm": 0.5583333333333333, "acc_norm_stderr": 0.009067881941319685, "acc_stderr": 0.009047063456897982}, "exercise": {"acc": 0.5706666666666667, "acc_norm": 0.5546666666666666, "acc_norm_stderr": 0.009075496684215473, "acc_stderr": 0.009038582451449423}, "valid_binary": {"acc": 0.551, "acc_norm": 0.5406666666666666, "acc_norm_stderr": 0.00909998226920486, "acc_stderr": 0.009082611478924382}, "yes_no_question": {"acc": 0.6023333333333334, "acc_norm": 0.6103333333333333, "acc_norm_stderr": 0.008905164372580985, "acc_stderr": 0.008936959925716909}}, "5": {"GPT-3 Style": {"acc": 0.5336666666666666, "acc_norm": 0.5676666666666667, "acc_norm_stderr": 0.009046234144187917, "acc_stderr": 0.009109510391803648}, "after_reading": {"acc": 0.5746666666666667, "acc_norm": 0.565, "acc_norm_stderr": 0.009052751926300883, "acc_stderr": 0.009027853030468729}, "exercise": {"acc": 0.5703333333333334, "acc_norm": 0.561, "acc_norm_stderr": 0.009062029213030573, "acc_stderr": 0.00903944935393088}, "valid_binary": {"acc": 0.5626666666666666, "acc_norm": 0.5513333333333333, "acc_norm_stderr": 0.009081985306932099, "acc_stderr": 0.00905823640921586}, "yes_no_question": {"acc": 0.6023333333333334, "acc_norm": 0.607, "acc_norm_stderr": 0.008918717088507564, "acc_stderr": 0.008936959925716905}}}, "cb": {"0": {"GPT-3 style": {"acc": 0.19642857142857142, "acc_stderr": 0.05357142857142859, "f1": 0.14285714285714288}, "MNLI crowdsource": {"acc": 0.4107142857142857, "acc_stderr": 0.06633634150359538, "f1": 0.1940928270042194}, "can we infer": {"acc": 0.5535714285714286, "acc_stderr": 0.06703189227942397, "f1": 0.3737373737373737}, "guaranteed/possible/impossible": {"acc": 0.23214285714285715, "acc_stderr": 0.056929390240001085, "f1": 0.18837606837606838}, "justified in saying": {"acc": 0.5357142857142857, "acc_stderr": 0.06724777654937658, "f1": 0.3671394799054373}}, "1": {"GPT-3 style": {"acc": 0.44642857142857145, "acc_stderr": 0.06703189227942398, "f1": 0.3171545968156138}, "MNLI crowdsource": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.2842025699168556}, "can we infer": {"acc": 0.4107142857142857, "acc_stderr": 0.06633634150359541, "f1": 0.29363017934446506}, "guaranteed/possible/impossible": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.2842025699168556}, "justified in saying": {"acc": 0.44642857142857145, "acc_stderr": 0.06703189227942398, "f1": 0.3172825681224338}}, "2": {"GPT-3 style": {"acc": 0.42857142857142855, "acc_stderr": 0.06672848092813058, "f1": 0.271744595274007}, "MNLI crowdsource": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.27010148321623734}, "can we infer": {"acc": 0.44642857142857145, "acc_stderr": 0.06703189227942398, "f1": 0.2956393200295639}, "guaranteed/possible/impossible": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.2807909604519774}, "justified in saying": {"acc": 0.44642857142857145, "acc_stderr": 0.06703189227942398, "f1": 0.2956393200295639}}, "3": {"GPT-3 style": {"acc": 0.4107142857142857, "acc_stderr": 0.0663363415035954, "f1": 0.24603174603174602}, "MNLI crowdsource": {"acc": 0.42857142857142855, "acc_stderr": 0.06672848092813058, "f1": 0.3048219682039154}, "can we infer": {"acc": 0.4107142857142857, "acc_stderr": 0.0663363415035954, "f1": 0.25302445302445303}, "guaranteed/possible/impossible": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.25852272727272724}, "justified in saying": {"acc": 0.4107142857142857, "acc_stderr": 0.0663363415035954, "f1": 0.25302445302445303}}, "4": {"GPT-3 style": {"acc": 0.42857142857142855, "acc_stderr": 0.06672848092813058, "f1": 0.261867501304121}, "MNLI crowdsource": {"acc": 0.5, "acc_stderr": 0.06741998624632421, "f1": 0.3325281803542673}, "can we infer": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.2593406593406593}, "guaranteed/possible/impossible": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.2719797596457938}, "justified in saying": {"acc": 0.42857142857142855, "acc_stderr": 0.06672848092813058, "f1": 0.27129417747266715}}, "5": {"GPT-3 style": {"acc": 0.39285714285714285, "acc_stderr": 0.0658538889806635, "f1": 0.235006435006435}, "MNLI crowdsource": {"acc": 0.4642857142857143, "acc_stderr": 0.0672477765493766, "f1": 0.28451178451178455}, "can we infer": {"acc": 0.42857142857142855, "acc_stderr": 0.06672848092813058, "f1": 0.2854700854700854}, "guaranteed/possible/impossible": {"acc": 0.4107142857142857, "acc_stderr": 0.06633634150359542, "f1": 0.26059456040480705}, "justified in saying": {"acc": 0.44642857142857145, "acc_stderr": 0.06703189227942398, "f1": 0.28883861236802416}}}, "copa": {"0": {"best_option": {"acc": 0.6, "acc_norm": 0.56, "acc_norm_stderr": 0.04988876515698589, "acc_stderr": 0.049236596391733084}, "cause_effect": {"acc": 0.56, "acc_norm": 0.52, "acc_norm_stderr": 0.050211673156867795, "acc_stderr": 0.049888765156985884}, "choose": {"acc": 0.59, "acc_norm": 0.49, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04943110704237102}, "i_am_hesitating": {"acc": 0.59, "acc_norm": 0.53, "acc_norm_stderr": 0.050161355804659205, "acc_stderr": 0.04943110704237102}, "plausible_alternatives": {"acc": 0.58, "acc_norm": 0.49, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.049604496374885836}}, "1": {"best_option": {"acc": 0.5, "acc_norm": 0.55, "acc_norm_stderr": 0.049999999999999996, "acc_stderr": 0.050251890762960605}, "cause_effect": {"acc": 0.57, "acc_norm": 0.47, "acc_norm_stderr": 0.05016135580465919, "acc_stderr": 0.049756985195624284}, "choose": {"acc": 0.57, "acc_norm": 0.49, "acc_norm_stderr": 0.05024183937956911, "acc_stderr": 0.04975698519562428}, "i_am_hesitating": {"acc": 0.55, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.049999999999999996}, "plausible_alternatives": {"acc": 0.56, "acc_norm": 0.51, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04988876515698589}}, "2": {"best_option": {"acc": 0.52, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.050211673156867795}, "cause_effect": {"acc": 0.56, "acc_norm": 0.51, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04988876515698589}, "choose": {"acc": 0.58, "acc_norm": 0.49, "acc_norm_stderr": 0.05024183937956911, "acc_stderr": 0.049604496374885836}, "i_am_hesitating": {"acc": 0.56, "acc_norm": 0.52, "acc_norm_stderr": 0.050211673156867795, "acc_stderr": 0.04988876515698589}, "plausible_alternatives": {"acc": 0.58, "acc_norm": 0.48, "acc_norm_stderr": 0.050211673156867795, "acc_stderr": 0.049604496374885836}}, "3": {"best_option": {"acc": 0.61, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.04902071300001975}, "cause_effect": {"acc": 0.59, "acc_norm": 0.51, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04943110704237102}, "choose": {"acc": 0.61, "acc_norm": 0.54, "acc_norm_stderr": 0.05009082659620333, "acc_stderr": 0.04902071300001975}, "i_am_hesitating": {"acc": 0.57, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.04975698519562428}, "plausible_alternatives": {"acc": 0.6, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.049236596391733084}}, "4": {"best_option": {"acc": 0.58, "acc_norm": 0.55, "acc_norm_stderr": 0.05, "acc_stderr": 0.049604496374885836}, "cause_effect": {"acc": 0.56, "acc_norm": 0.47, "acc_norm_stderr": 0.05016135580465919, "acc_stderr": 0.04988876515698589}, "choose": {"acc": 0.59, "acc_norm": 0.53, "acc_norm_stderr": 0.050161355804659205, "acc_stderr": 0.04943110704237102}, "i_am_hesitating": {"acc": 0.56, "acc_norm": 0.49, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04988876515698589}, "plausible_alternatives": {"acc": 0.55, "acc_norm": 0.48, "acc_norm_stderr": 0.050211673156867795, "acc_stderr": 0.05}}, "5": {"best_option": {"acc": 0.58, "acc_norm": 0.53, "acc_norm_stderr": 0.050161355804659205, "acc_stderr": 0.049604496374885836}, "cause_effect": {"acc": 0.55, "acc_norm": 0.5, "acc_norm_stderr": 0.050251890762960605, "acc_stderr": 0.05}, "choose": {"acc": 0.57, "acc_norm": 0.51, "acc_norm_stderr": 0.05024183937956912, "acc_stderr": 0.04975698519562427}, "i_am_hesitating": {"acc": 0.57, "acc_norm": 0.46, "acc_norm_stderr": 0.05009082659620332, "acc_stderr": 0.04975698519562427}, "plausible_alternatives": {"acc": 0.56, "acc_norm": 0.45, "acc_norm_stderr": 0.04999999999999999, "acc_stderr": 0.049888765156985884}}}, "e2e_nlg_cleaned": {"0": {"coherent_text": {"bleu": 1.5651118840944627, "bleu_stderr": 0.042981320396581084, "rouge1_fmeasure": 0.21383961074239713, "rouge1_fmeasure_stderr": 0.0026815356045279183, "rouge1_precision": 0.18521451898205324, "rouge1_precision_stderr": 0.002457080525787092, "rouge1_recall": 0.27275459055962076, "rouge1_recall_stderr": 0.00326239965656717, "rouge2_fmeasure": 0.07683031179945712, "rouge2_fmeasure_stderr": 0.0014361637959827516, "rouge2_precision": 0.06742918538311436, "rouge2_precision_stderr": 0.0012528926958132866, "rouge2_recall": 0.0951561319414526, "rouge2_recall_stderr": 0.0018453850908383695, "rougeL_fmeasure": 0.18671896323697262, "rougeL_fmeasure_stderr": 0.002076974744205931, "rougeL_precision": 0.16070278930701914, "rougeL_precision_stderr": 0.00185948984697632, "rougeL_recall": 0.24114365421062722, "rougeL_recall_stderr": 0.0026802570784209123, "rougeLsum_fmeasure": 0.1825401908962132, "rougeLsum_fmeasure_stderr": 0.0021610595182852883, "rougeLsum_precision": 0.15871609393029643, "rougeLsum_precision_stderr": 0.0020022541432733187, "rougeLsum_recall": 0.23173214485891466, "rougeLsum_recall_stderr": 0.0026053790108179174}, "create_text_for_me": {"bleu": 2.604286337933278, "bleu_stderr": 0.03591674116891077, "rouge1_fmeasure": 0.30133223661624636, "rouge1_fmeasure_stderr": 0.0024851056944864957, "rouge1_precision": 0.2680956372841705, "rouge1_precision_stderr": 0.0022592902002292296, "rouge1_recall": 0.370826122128093, "rouge1_recall_stderr": 0.0035950144885036634, "rouge2_fmeasure": 0.12592500811764892, "rouge2_fmeasure_stderr": 0.0013089159002261316, "rouge2_precision": 0.11163002440463841, "rouge2_precision_stderr": 0.0011657527940780504, "rouge2_recall": 0.15654465364549336, "rouge2_recall_stderr": 0.0018850260109515933, "rougeL_fmeasure": 0.2329565620227209, "rougeL_fmeasure_stderr": 0.001687724446352596, "rougeL_precision": 0.20852815533900124, "rougeL_precision_stderr": 0.0016092176061611565, "rougeL_recall": 0.2840330760519545, "rougeL_recall_stderr": 0.0024031245269283963, "rougeLsum_fmeasure": 0.2633236447094561, "rougeLsum_fmeasure_stderr": 0.0020214976908264665, "rougeLsum_precision": 0.2355692548868321, "rougeLsum_precision_stderr": 0.00191083615835956, "rougeLsum_recall": 0.32127109053431796, "rougeLsum_recall_stderr": 0.0028423630075748696}, "generate_gramatically_correct_text": {"bleu": 0.5554048543804149, "bleu_stderr": 0.04843400111692929, "rouge1_fmeasure": 0.06971399215838711, "rouge1_fmeasure_stderr": 0.002110647717043094, "rouge1_precision": 0.07490676533270745, "rouge1_precision_stderr": 0.002150230516202816, "rouge1_recall": 0.06832821832134156, "rouge1_recall_stderr": 0.0022347623309728333, "rouge2_fmeasure": 0.03866825902601929, "rouge2_fmeasure_stderr": 0.001277939861242296, "rouge2_precision": 0.04122768230311284, "rouge2_precision_stderr": 0.0012955576500527062, "rouge2_recall": 0.03832509183940306, "rouge2_recall_stderr": 0.0013668800475287908, "rougeL_fmeasure": 0.06647105884856924, "rougeL_fmeasure_stderr": 0.0018913981605438715, "rougeL_precision": 0.07153482505420374, "rougeL_precision_stderr": 0.0019220236379015598, "rougeL_recall": 0.06510051970164941, "rougeL_recall_stderr": 0.002019905690872576, "rougeLsum_fmeasure": 0.06598366934280134, "rougeLsum_fmeasure_stderr": 0.002014393055687619, "rougeLsum_precision": 0.07082417678060954, "rougeLsum_precision_stderr": 0.002048106631259025, "rougeLsum_recall": 0.06475551139934367, "rougeLsum_recall_stderr": 0.0021371018812695345}, "generate_text_restaurant": {"bleu": 0.5379698042344059, "bleu_stderr": 0.023682890094100795, "rouge1_fmeasure": 0.13249245505272914, "rouge1_fmeasure_stderr": 0.0023865964860820023, "rouge1_precision": 0.11184434244818633, "rouge1_precision_stderr": 0.0020578473204419783, "rouge1_recall": 0.17494853797920373, "rouge1_recall_stderr": 0.003281205101820411, "rouge2_fmeasure": 0.02216822438561205, "rouge2_fmeasure_stderr": 0.0009397403776169823, "rouge2_precision": 0.018039929895774978, "rouge2_precision_stderr": 0.0007516851699163001, "rouge2_recall": 0.03139481036037028, "rouge2_recall_stderr": 0.0014237434012642758, "rougeL_fmeasure": 0.09481463004436187, "rougeL_fmeasure_stderr": 0.001541902659571312, "rougeL_precision": 0.07937245443292806, "rougeL_precision_stderr": 0.0013055330499913698, "rougeL_recall": 0.1269667545228582, "rougeL_recall_stderr": 0.0022135782119190543, "rougeLsum_fmeasure": 0.12378548376730361, "rougeLsum_fmeasure_stderr": 0.002209146731463173, "rougeLsum_precision": 0.10436515332917147, "rougeLsum_precision_stderr": 0.0018952448699960476, "rougeLsum_recall": 0.16392541157802315, "rougeLsum_recall_stderr": 0.003068122098698215}, "text": {"bleu": 3.416746947803346, "bleu_stderr": 0.04915567383423248, "rouge1_fmeasure": 0.43965168599123006, "rouge1_fmeasure_stderr": 0.0015705219676545596, "rouge1_precision": 0.41219549454005344, "rouge1_precision_stderr": 0.0019884907431696104, "rouge1_recall": 0.49154498554014486, "rouge1_recall_stderr": 0.0018555667598375688, "rouge2_fmeasure": 0.18707996026648183, "rouge2_fmeasure_stderr": 0.0010713647179164203, "rouge2_precision": 0.17378317584242825, "rouge2_precision_stderr": 0.0010901893538702665, "rouge2_recall": 0.21285050587435922, "rouge2_recall_stderr": 0.001428926493666674, "rougeL_fmeasure": 0.33221821033645077, "rougeL_fmeasure_stderr": 0.0011795568835039534, "rougeL_precision": 0.3073557477422296, "rougeL_precision_stderr": 0.0011789635478387738, "rougeL_recall": 0.37811034759792467, "rougeL_recall_stderr": 0.0019549694924619787, "rougeLsum_fmeasure": 0.3638009261252252, "rougeLsum_fmeasure_stderr": 0.0013481348026921308, "rougeLsum_precision": 0.33947329485158056, "rougeLsum_precision_stderr": 0.0015891394303630454, "rougeLsum_recall": 0.40971450190173514, "rougeLsum_recall_stderr": 0.0018376040861944267}}, "1": {"coherent_text": {"bleu": 1.990645378245907, "bleu_stderr": 0.09426201816772427, "rouge1_fmeasure": 0.17658380722184017, "rouge1_fmeasure_stderr": 0.0031889753436656837, "rouge1_precision": 0.19474596234384664, "rouge1_precision_stderr": 0.0036808504078335246, "rouge1_recall": 0.17762533937937974, "rouge1_recall_stderr": 0.0033135503178414513, "rouge2_fmeasure": 0.04696802025613863, "rouge2_fmeasure_stderr": 0.0012839248316351875, "rouge2_precision": 0.05236928138258151, "rouge2_precision_stderr": 0.0015077160489151036, "rouge2_recall": 0.04723848634979288, "rouge2_recall_stderr": 0.0013288114505525092, "rougeL_fmeasure": 0.12661225264501008, "rougeL_fmeasure_stderr": 0.0023435534892212776, "rougeL_precision": 0.1402837628671216, "rougeL_precision_stderr": 0.002748825869859068, "rougeL_recall": 0.12751328685413893, "rougeL_recall_stderr": 0.002450859209100979, "rougeLsum_fmeasure": 0.14576477572483743, "rougeLsum_fmeasure_stderr": 0.002699050142839919, "rougeLsum_precision": 0.16127567452387717, "rougeLsum_precision_stderr": 0.003143964896409528, "rougeLsum_recall": 0.14650506662968157, "rougeLsum_recall_stderr": 0.0027996830850509635}, "create_text_for_me": {"bleu": 2.376958494986839, "bleu_stderr": 0.09315943625573786, "rouge1_fmeasure": 0.19177598419259367, "rouge1_fmeasure_stderr": 0.0031175904013624814, "rouge1_precision": 0.2142240439442094, "rouge1_precision_stderr": 0.003669585143669259, "rouge1_recall": 0.19274042573599687, "rouge1_recall_stderr": 0.0032419711801001993, "rouge2_fmeasure": 0.05060323801799519, "rouge2_fmeasure_stderr": 0.0012959436966401268, "rouge2_precision": 0.05683805777177476, "rouge2_precision_stderr": 0.0015258163112187955, "rouge2_recall": 0.05073039697609069, "rouge2_recall_stderr": 0.0013354368045998223, "rougeL_fmeasure": 0.1385854882700382, "rougeL_fmeasure_stderr": 0.0022957592520056066, "rougeL_precision": 0.1553382299379965, "rougeL_precision_stderr": 0.002742888570452596, "rougeL_recall": 0.13961389757181983, "rougeL_recall_stderr": 0.0024018984589737314, "rougeLsum_fmeasure": 0.15921566238118462, "rougeLsum_fmeasure_stderr": 0.002650725044809855, "rougeLsum_precision": 0.17863698031283276, "rougeLsum_precision_stderr": 0.003158636767078994, "rougeLsum_recall": 0.15979139572794768, "rougeLsum_recall_stderr": 0.0027474819161167415}, "generate_gramatically_correct_text": {"bleu": 3.1303501122849213, "bleu_stderr": 0.061431179427438594, "rouge1_fmeasure": 0.17861417240710098, "rouge1_fmeasure_stderr": 0.001841118798739779, "rouge1_precision": 0.18795241249491457, "rouge1_precision_stderr": 0.0019672359252996805, "rouge1_recall": 0.1822785883610696, "rouge1_recall_stderr": 0.0021558750964836095, "rouge2_fmeasure": 0.08617713780376496, "rouge2_fmeasure_stderr": 0.001215937184069791, "rouge2_precision": 0.09099657414603357, "rouge2_precision_stderr": 0.0012717971723587007, "rouge2_recall": 0.08699074354785269, "rouge2_recall_stderr": 0.0013450179899541406, "rougeL_fmeasure": 0.16060461647433702, "rougeL_fmeasure_stderr": 0.0013748351598955968, "rougeL_precision": 0.1695073282851938, "rougeL_precision_stderr": 0.0014742554265132549, "rougeL_recall": 0.16297998873383596, "rougeL_recall_stderr": 0.0016527556204933523, "rougeLsum_fmeasure": 0.1620576843620918, "rougeLsum_fmeasure_stderr": 0.0016013095501406327, "rougeLsum_precision": 0.17092043784842698, "rougeLsum_precision_stderr": 0.0017210240122471166, "rougeLsum_recall": 0.16483164473288184, "rougeLsum_recall_stderr": 0.001869983093761878}, "generate_text_restaurant": {"bleu": 3.711647887806667, "bleu_stderr": 0.12239209464334602, "rouge1_fmeasure": 0.28795658431823945, "rouge1_fmeasure_stderr": 0.001986436700440943, "rouge1_precision": 0.3347678963130461, "rouge1_precision_stderr": 0.002538034125678728, "rouge1_recall": 0.27699070131591086, "rouge1_recall_stderr": 0.0022531459521912115, "rouge2_fmeasure": 0.07230219657399567, "rouge2_fmeasure_stderr": 0.0013203982837554223, "rouge2_precision": 0.08380014964523355, "rouge2_precision_stderr": 0.0015843319722248113, "rouge2_recall": 0.06976453338241907, "rouge2_recall_stderr": 0.0013297329518253997, "rougeL_fmeasure": 0.2088938861271432, "rougeL_fmeasure_stderr": 0.001534187541306792, "rougeL_precision": 0.2439230596700563, "rougeL_precision_stderr": 0.0020005873112574108, "rougeL_recall": 0.20072421941893917, "rougeL_recall_stderr": 0.0017248535505826324, "rougeLsum_fmeasure": 0.2373352341595687, "rougeLsum_fmeasure_stderr": 0.0017982865567598523, "rougeLsum_precision": 0.27665907100653375, "rougeLsum_precision_stderr": 0.0023050480585854313, "rougeLsum_recall": 0.22794132690389687, "rougeLsum_recall_stderr": 0.001983838704481844}, "text": {"bleu": 3.8370970182392985, "bleu_stderr": 0.1144591225447721, "rouge1_fmeasure": 0.2885492968351212, "rouge1_fmeasure_stderr": 0.0020292469178824456, "rouge1_precision": 0.33744773531264344, "rouge1_precision_stderr": 0.002678432208002564, "rouge1_recall": 0.2779081994219456, "rouge1_recall_stderr": 0.002272364632457066, "rouge2_fmeasure": 0.0752688547278534, "rouge2_fmeasure_stderr": 0.0013552863861086071, "rouge2_precision": 0.08918695736438806, "rouge2_precision_stderr": 0.0017352721360830848, "rouge2_recall": 0.07226568712760938, "rouge2_recall_stderr": 0.0013494607128344402, "rougeL_fmeasure": 0.21099118017935053, "rougeL_fmeasure_stderr": 0.0015717685572481305, "rougeL_precision": 0.24853209420474298, "rougeL_precision_stderr": 0.002175861850486939, "rougeL_recall": 0.20284289627233723, "rougeL_recall_stderr": 0.0017347865605118378, "rougeLsum_fmeasure": 0.23909437838411668, "rougeLsum_fmeasure_stderr": 0.0018320148819675503, "rougeLsum_precision": 0.2808597038267283, "rougeLsum_precision_stderr": 0.0024612171563718217, "rougeLsum_recall": 0.2298552776553825, "rougeLsum_recall_stderr": 0.0019962013821171482}}, "2": {"coherent_text": {"bleu": 3.8652987845777536, "bleu_stderr": 0.10450428506636379, "rouge1_fmeasure": 0.2629920038581495, "rouge1_fmeasure_stderr": 0.002789298885039384, "rouge1_precision": 0.2928603436655027, "rouge1_precision_stderr": 0.0033407712098847088, "rouge1_recall": 0.26267840583267554, "rouge1_recall_stderr": 0.002974574226301873, "rouge2_fmeasure": 0.07274573138257315, "rouge2_fmeasure_stderr": 0.0014344798818270634, "rouge2_precision": 0.08172049965690244, "rouge2_precision_stderr": 0.001717030331861244, "rouge2_recall": 0.07276650009557936, "rouge2_recall_stderr": 0.0014807864762006147, "rougeL_fmeasure": 0.19024237456492324, "rougeL_fmeasure_stderr": 0.002100380195663744, "rougeL_precision": 0.21347346743709447, "rougeL_precision_stderr": 0.0026042923393898563, "rougeL_recall": 0.1899975461270108, "rougeL_recall_stderr": 0.0022474664670417152, "rougeLsum_fmeasure": 0.2176826658743973, "rougeLsum_fmeasure_stderr": 0.0024213633519383406, "rougeLsum_precision": 0.24332907877165075, "rougeLsum_precision_stderr": 0.0029356367554022487, "rougeLsum_recall": 0.21726288370114252, "rougeLsum_recall_stderr": 0.002572105790167164}, "create_text_for_me": {"bleu": 4.029805304404254, "bleu_stderr": 0.09060124588998114, "rouge1_fmeasure": 0.27787798890903986, "rouge1_fmeasure_stderr": 0.0025362776452563375, "rouge1_precision": 0.3146026140303434, "rouge1_precision_stderr": 0.003123666505855817, "rouge1_recall": 0.27489622295236277, "rouge1_recall_stderr": 0.0027426307774660475, "rouge2_fmeasure": 0.07551724406985702, "rouge2_fmeasure_stderr": 0.001406834249268174, "rouge2_precision": 0.08627383555029068, "rouge2_precision_stderr": 0.0017285779836978593, "rouge2_recall": 0.07473352232906243, "rouge2_recall_stderr": 0.001444402605968691, "rougeL_fmeasure": 0.20227021958830982, "rougeL_fmeasure_stderr": 0.0019281583412694655, "rougeL_precision": 0.23138204856779457, "rougeL_precision_stderr": 0.0025050345781738757, "rougeL_recall": 0.1997296995843297, "rougeL_recall_stderr": 0.002078456692172228, "rougeLsum_fmeasure": 0.23048345042133192, "rougeLsum_fmeasure_stderr": 0.0022174670681384323, "rougeLsum_precision": 0.26264908396487563, "rougeLsum_precision_stderr": 0.002801395123048888, "rougeLsum_recall": 0.22749287503171903, "rougeLsum_recall_stderr": 0.002372211809509306}, "generate_gramatically_correct_text": {"bleu": 3.3001775192249134, "bleu_stderr": 0.10184239170439335, "rouge1_fmeasure": 0.20055771804060046, "rouge1_fmeasure_stderr": 0.0021701685544756343, "rouge1_precision": 0.2134701829367398, "rouge1_precision_stderr": 0.0024468758692696006, "rouge1_recall": 0.20530621471645766, "rouge1_recall_stderr": 0.002460921154929664, "rouge2_fmeasure": 0.08158846564278203, "rouge2_fmeasure_stderr": 0.0012860200593204027, "rouge2_precision": 0.08715705739965049, "rouge2_precision_stderr": 0.0014291170688220639, "rouge2_recall": 0.08282189480619162, "rouge2_recall_stderr": 0.0014083407755700517, "rougeL_fmeasure": 0.16927544675510284, "rougeL_fmeasure_stderr": 0.0015459458010864233, "rougeL_precision": 0.18060729349255075, "rougeL_precision_stderr": 0.0017814219442781176, "rougeL_recall": 0.17257800926899938, "rougeL_recall_stderr": 0.0018037731355666897, "rougeLsum_fmeasure": 0.1759439032072785, "rougeLsum_fmeasure_stderr": 0.0018563724554560355, "rougeLsum_precision": 0.18766246584705398, "rougeLsum_precision_stderr": 0.00211000963150373, "rougeLsum_recall": 0.1797767907757674, "rougeLsum_recall_stderr": 0.002115732933379051}, "generate_text_restaurant": {"bleu": 4.158981847733467, "bleu_stderr": 0.10805920566927497, "rouge1_fmeasure": 0.296355770492695, "rouge1_fmeasure_stderr": 0.002043539678118385, "rouge1_precision": 0.3434326875335696, "rouge1_precision_stderr": 0.0026992899622732525, "rouge1_recall": 0.2865272615258301, "rouge1_recall_stderr": 0.002262834986590033, "rouge2_fmeasure": 0.07905711793730946, "rouge2_fmeasure_stderr": 0.0013774541250299033, "rouge2_precision": 0.09254926113645746, "rouge2_precision_stderr": 0.0017008623486038437, "rouge2_recall": 0.0763624864031587, "rouge2_recall_stderr": 0.0013832865499576604, "rougeL_fmeasure": 0.216815482854756, "rougeL_fmeasure_stderr": 0.0016012856900651986, "rougeL_precision": 0.25255081340500835, "rougeL_precision_stderr": 0.002168002738840324, "rougeL_recall": 0.20966503552217744, "rougeL_recall_stderr": 0.0017709941856742383, "rougeLsum_fmeasure": 0.244683933643628, "rougeLsum_fmeasure_stderr": 0.0018556926092760141, "rougeLsum_precision": 0.2842764554431881, "rougeLsum_precision_stderr": 0.0024381683573090783, "rougeLsum_recall": 0.23649066236480792, "rougeLsum_recall_stderr": 0.002022905140524907}, "text": {"bleu": 4.373718657948969, "bleu_stderr": 0.11887879947848155, "rouge1_fmeasure": 0.29973019758970454, "rouge1_fmeasure_stderr": 0.0020502121064942818, "rouge1_precision": 0.3522552603743486, "rouge1_precision_stderr": 0.002763858033996764, "rouge1_recall": 0.2889714759845472, "rouge1_recall_stderr": 0.00228492884684045, "rouge2_fmeasure": 0.0831703337253987, "rouge2_fmeasure_stderr": 0.0014123449794312788, "rouge2_precision": 0.09934840296950617, "rouge2_precision_stderr": 0.0018189448808929704, "rouge2_recall": 0.07999739923020184, "rouge2_recall_stderr": 0.0014210909684714815, "rougeL_fmeasure": 0.22110973690939364, "rougeL_fmeasure_stderr": 0.0016159023437830125, "rougeL_precision": 0.26226475143890166, "rougeL_precision_stderr": 0.002301455268301791, "rougeL_recall": 0.21294988420936256, "rougeL_recall_stderr": 0.0017815097701725465, "rougeLsum_fmeasure": 0.24955436884636814, "rougeLsum_fmeasure_stderr": 0.0018809736592006266, "rougeLsum_precision": 0.2946849915708113, "rougeLsum_precision_stderr": 0.0025524376570338707, "rougeLsum_recall": 0.24032554668210832, "rougeLsum_recall_stderr": 0.002050233303327457}}, "3": {"coherent_text": {"bleu": 4.749488853686958, "bleu_stderr": 0.1439737171039592, "rouge1_fmeasure": 0.29307632394991145, "rouge1_fmeasure_stderr": 0.0024677702395856805, "rouge1_precision": 0.3295110755139736, "rouge1_precision_stderr": 0.0030466187495971652, "rouge1_recall": 0.2908170186604444, "rouge1_recall_stderr": 0.002713234606538018, "rouge2_fmeasure": 0.08511598673947676, "rouge2_fmeasure_stderr": 0.0014976664233839329, "rouge2_precision": 0.09691059758420743, "rouge2_precision_stderr": 0.0018267761852325394, "rouge2_recall": 0.08477572813016075, "rouge2_recall_stderr": 0.0015697017492672485, "rougeL_fmeasure": 0.21289346624936825, "rougeL_fmeasure_stderr": 0.0018914326319727048, "rougeL_precision": 0.24119075971078466, "rougeL_precision_stderr": 0.0024282366700050326, "rougeL_recall": 0.21098836672574872, "rougeL_recall_stderr": 0.002067666204626933, "rougeLsum_fmeasure": 0.24293590407668367, "rougeLsum_fmeasure_stderr": 0.002178523714675789, "rougeLsum_precision": 0.274291969853511, "rougeLsum_precision_stderr": 0.0027222499154415615, "rougeLsum_recall": 0.24075001746941485, "rougeLsum_recall_stderr": 0.002369489900790716}, "create_text_for_me": {"bleu": 4.498804726055727, "bleu_stderr": 0.10775002870645801, "rouge1_fmeasure": 0.2968322137022183, "rouge1_fmeasure_stderr": 0.0022769902890191806, "rouge1_precision": 0.3356969876826268, "rouge1_precision_stderr": 0.0028462582702245168, "rouge1_recall": 0.2941317708277562, "rouge1_recall_stderr": 0.0025456085630560714, "rouge2_fmeasure": 0.08225064700727518, "rouge2_fmeasure_stderr": 0.0014348926547678262, "rouge2_precision": 0.0935059203110401, "rouge2_precision_stderr": 0.0017602062083097658, "rouge2_recall": 0.0818949185573383, "rouge2_recall_stderr": 0.0014958409085964503, "rougeL_fmeasure": 0.21588071480185272, "rougeL_fmeasure_stderr": 0.0017373391626468349, "rougeL_precision": 0.24614076393148132, "rougeL_precision_stderr": 0.002283618951318419, "rougeL_recall": 0.21389679836058903, "rougeL_recall_stderr": 0.0019512939427380196, "rougeLsum_fmeasure": 0.24604509499852464, "rougeLsum_fmeasure_stderr": 0.002021644244552477, "rougeLsum_precision": 0.27936714820247016, "rougeLsum_precision_stderr": 0.0025524294154527605, "rougeLsum_recall": 0.24370834239777953, "rougeLsum_recall_stderr": 0.002246504322912202}, "generate_gramatically_correct_text": {"bleu": 3.666475893279001, "bleu_stderr": 0.10987537074629235, "rouge1_fmeasure": 0.21946650594900366, "rouge1_fmeasure_stderr": 0.0024037209981410657, "rouge1_precision": 0.23836199838819472, "rouge1_precision_stderr": 0.0028469068712546316, "rouge1_recall": 0.22275349279619636, "rouge1_recall_stderr": 0.002634091620880004, "rouge2_fmeasure": 0.07939963217690697, "rouge2_fmeasure_stderr": 0.0013776332563822653, "rouge2_precision": 0.08677164701585942, "rouge2_precision_stderr": 0.001628328073416126, "rouge2_recall": 0.08034781449611338, "rouge2_recall_stderr": 0.001485529414823335, "rougeL_fmeasure": 0.1775636766730472, "rougeL_fmeasure_stderr": 0.0016879677128570562, "rougeL_precision": 0.1931177821473266, "rougeL_precision_stderr": 0.0020670381912188582, "rougeL_recall": 0.18001529185227896, "rougeL_recall_stderr": 0.0019058042973943167, "rougeLsum_fmeasure": 0.1882137601760175, "rougeLsum_fmeasure_stderr": 0.0020477882273726594, "rougeLsum_precision": 0.20467508975739554, "rougeLsum_precision_stderr": 0.0024391669746580877, "rougeLsum_recall": 0.19090226151707998, "rougeLsum_recall_stderr": 0.002261425815356093}, "generate_text_restaurant": {"bleu": 4.7695767170826615, "bleu_stderr": 0.16549584151536006, "rouge1_fmeasure": 0.3035075966249268, "rouge1_fmeasure_stderr": 0.002031043294823014, "rouge1_precision": 0.35625735674057685, "rouge1_precision_stderr": 0.002753457728280898, "rouge1_recall": 0.29265236837044695, "rouge1_recall_stderr": 0.002280330795443891, "rouge2_fmeasure": 0.08864387470534706, "rouge2_fmeasure_stderr": 0.001446455854434278, "rouge2_precision": 0.10577742526833883, "rouge2_precision_stderr": 0.0018673396392927667, "rouge2_recall": 0.08549991950256323, "rouge2_recall_stderr": 0.0014672056887335024, "rougeL_fmeasure": 0.2260702121301555, "rougeL_fmeasure_stderr": 0.0016207565003713668, "rougeL_precision": 0.2671981244809395, "rougeL_precision_stderr": 0.002307047675439304, "rougeL_recall": 0.21813281187586306, "rougeL_recall_stderr": 0.0018116496367021734, "rougeLsum_fmeasure": 0.2541669303215731, "rougeLsum_fmeasure_stderr": 0.0018585514707850925, "rougeLsum_precision": 0.29969790030371246, "rougeLsum_precision_stderr": 0.002551276392318688, "rougeLsum_recall": 0.24481991830895322, "rougeLsum_recall_stderr": 0.0020451634619149594}, "text": {"bleu": 4.872872536054477, "bleu_stderr": 0.17994394529095062, "rouge1_fmeasure": 0.3055513704733046, "rouge1_fmeasure_stderr": 0.0020673466657850498, "rouge1_precision": 0.360735661960322, "rouge1_precision_stderr": 0.002835390448997847, "rouge1_recall": 0.2944152190187286, "rouge1_recall_stderr": 0.0023207814101633794, "rouge2_fmeasure": 0.09057857385566902, "rouge2_fmeasure_stderr": 0.0014742462627991503, "rouge2_precision": 0.10920157678214337, "rouge2_precision_stderr": 0.0019434279841216329, "rouge2_recall": 0.0870860469506747, "rouge2_recall_stderr": 0.0014872544127445687, "rougeL_fmeasure": 0.2287609415460662, "rougeL_fmeasure_stderr": 0.0016617607367168975, "rougeL_precision": 0.27244358337055397, "rougeL_precision_stderr": 0.0024054386143698534, "rougeL_recall": 0.22030127184406081, "rougeL_recall_stderr": 0.0018506185371882098, "rougeLsum_fmeasure": 0.25668504598663233, "rougeLsum_fmeasure_stderr": 0.0018889983272758794, "rougeLsum_precision": 0.30429745199273023, "rougeLsum_precision_stderr": 0.002612593595236828, "rougeLsum_recall": 0.2470915077136621, "rougeLsum_recall_stderr": 0.002085484889135651}}, "4": {"coherent_text": {"bleu": 5.176301731528647, "bleu_stderr": 0.14682428099423225, "rouge1_fmeasure": 0.3042294531279061, "rouge1_fmeasure_stderr": 0.002297980823161359, "rouge1_precision": 0.34370313232143146, "rouge1_precision_stderr": 0.002947894794228507, "rouge1_recall": 0.3028195228554386, "rouge1_recall_stderr": 0.002584602527686234, "rouge2_fmeasure": 0.0903654887337426, "rouge2_fmeasure_stderr": 0.0015331774140601283, "rouge2_precision": 0.10396227858967162, "rouge2_precision_stderr": 0.0019360112695293232, "rouge2_recall": 0.09038662176400038, "rouge2_recall_stderr": 0.0016265630560554562, "rougeL_fmeasure": 0.2217428897230324, "rougeL_fmeasure_stderr": 0.0017992319739905335, "rougeL_precision": 0.2524069371766252, "rougeL_precision_stderr": 0.002404139964914726, "rougeL_recall": 0.2208918495013449, "rougeL_recall_stderr": 0.0020294042523474267, "rougeLsum_fmeasure": 0.25331485964949535, "rougeLsum_fmeasure_stderr": 0.0020731222812007586, "rougeLsum_precision": 0.28720158613196195, "rougeLsum_precision_stderr": 0.002676104542866642, "rougeLsum_recall": 0.2520155455904404, "rougeLsum_recall_stderr": 0.0023002075133208935}, "create_text_for_me": {"bleu": 4.927139553620482, "bleu_stderr": 0.12665338005793328, "rouge1_fmeasure": 0.30772601506425196, "rouge1_fmeasure_stderr": 0.0021433656889168397, "rouge1_precision": 0.34744584188483457, "rouge1_precision_stderr": 0.0027671133325864244, "rouge1_recall": 0.30598680841317655, "rouge1_recall_stderr": 0.0024605634807434787, "rouge2_fmeasure": 0.08832498480105934, "rouge2_fmeasure_stderr": 0.0014651207993424452, "rouge2_precision": 0.10081802718812478, "rouge2_precision_stderr": 0.0017973283857860576, "rouge2_recall": 0.08800270112485714, "rouge2_recall_stderr": 0.0015444421661352863, "rougeL_fmeasure": 0.2249364821771828, "rougeL_fmeasure_stderr": 0.0016844326994501068, "rougeL_precision": 0.2559711012561957, "rougeL_precision_stderr": 0.0022826839200581384, "rougeL_recall": 0.2237171867711202, "rougeL_recall_stderr": 0.001933235277196951, "rougeLsum_fmeasure": 0.2564962774637448, "rougeLsum_fmeasure_stderr": 0.0019326551590934662, "rougeLsum_precision": 0.2906537194445758, "rougeLsum_precision_stderr": 0.0025152309187473194, "rougeLsum_recall": 0.2548745214297313, "rougeLsum_recall_stderr": 0.0021861203156797256}, "generate_gramatically_correct_text": {"bleu": 3.8809334302287417, "bleu_stderr": 0.13581356578783566, "rouge1_fmeasure": 0.23609371210606708, "rouge1_fmeasure_stderr": 0.002433429420325999, "rouge1_precision": 0.26018033627096865, "rouge1_precision_stderr": 0.0029849403702905605, "rouge1_recall": 0.23824263498663878, "rouge1_recall_stderr": 0.0026488517402878512, "rouge2_fmeasure": 0.07785610993313638, "rouge2_fmeasure_stderr": 0.0013788846901543758, "rouge2_precision": 0.08611475286824664, "rouge2_precision_stderr": 0.001635778374326644, "rouge2_recall": 0.07858583110015212, "rouge2_recall_stderr": 0.0014801568276794114, "rougeL_fmeasure": 0.18500764514280799, "rougeL_fmeasure_stderr": 0.0017083269070961805, "rougeL_precision": 0.20412964784769164, "rougeL_precision_stderr": 0.0021807066507413034, "rougeL_recall": 0.18679664087085932, "rougeL_recall_stderr": 0.0019178081474592414, "rougeLsum_fmeasure": 0.19959413484150632, "rougeLsum_fmeasure_stderr": 0.002096279733479644, "rougeLsum_precision": 0.22044880753033722, "rougeLsum_precision_stderr": 0.002598387740639484, "rougeLsum_recall": 0.20134137612920308, "rougeLsum_recall_stderr": 0.002286825517593027}, "generate_text_restaurant": {"bleu": 5.0994041530025545, "bleu_stderr": 0.15184509857797265, "rouge1_fmeasure": 0.3069669703069319, "rouge1_fmeasure_stderr": 0.002055596225803597, "rouge1_precision": 0.3625581772528726, "rouge1_precision_stderr": 0.002820926829290532, "rouge1_recall": 0.29556687359417316, "rouge1_recall_stderr": 0.0023030954598190034, "rouge2_fmeasure": 0.09381098739864982, "rouge2_fmeasure_stderr": 0.0015224351850222366, "rouge2_precision": 0.11265300538187106, "rouge2_precision_stderr": 0.001984841845142737, "rouge2_recall": 0.0904636966530554, "rouge2_recall_stderr": 0.001544958177720571, "rougeL_fmeasure": 0.22975500104331817, "rougeL_fmeasure_stderr": 0.0016770985431782879, "rougeL_precision": 0.2731899620101324, "rougeL_precision_stderr": 0.00239084885412292, "rougeL_recall": 0.22137993241122347, "rougeL_recall_stderr": 0.0018642397142735888, "rougeLsum_fmeasure": 0.25753199178665565, "rougeLsum_fmeasure_stderr": 0.0019112760465508692, "rougeLsum_precision": 0.30531248482102935, "rougeLsum_precision_stderr": 0.0026231141349939897, "rougeLsum_recall": 0.24781445089520615, "rougeLsum_recall_stderr": 0.002101182002409543}, "text": {"bleu": 5.212454694507268, "bleu_stderr": 0.1288163730607949, "rouge1_fmeasure": 0.3090786443943064, "rouge1_fmeasure_stderr": 0.0020670761005209533, "rouge1_precision": 0.36456574097223493, "rouge1_precision_stderr": 0.002845913293926449, "rouge1_recall": 0.2989078546481851, "rouge1_recall_stderr": 0.0023201439477763606, "rouge2_fmeasure": 0.0944527466973476, "rouge2_fmeasure_stderr": 0.001527049255606511, "rouge2_precision": 0.11386196869648488, "rouge2_precision_stderr": 0.0020442207769660018, "rouge2_recall": 0.09115599764568338, "rouge2_recall_stderr": 0.0015501355013158944, "rougeL_fmeasure": 0.2323973425908195, "rougeL_fmeasure_stderr": 0.0016905329597190666, "rougeL_precision": 0.2769682674664966, "rougeL_precision_stderr": 0.002470609873235701, "rougeL_recall": 0.22450967972348715, "rougeL_recall_stderr": 0.0018729810994751136, "rougeLsum_fmeasure": 0.26051491605603, "rougeLsum_fmeasure_stderr": 0.0019072972536883094, "rougeLsum_precision": 0.3089928251978803, "rougeLsum_precision_stderr": 0.002660795958922366, "rougeLsum_recall": 0.2515644789803418, "rougeLsum_recall_stderr": 0.002098522007442167}}, "5": {"coherent_text": {"bleu": 5.448542486611392, "bleu_stderr": 0.1576603370541913, "rouge1_fmeasure": 0.3118801144518462, "rouge1_fmeasure_stderr": 0.002229281342893514, "rouge1_precision": 0.3534259465453393, "rouge1_precision_stderr": 0.002910909198697379, "rouge1_recall": 0.31004812770308093, "rouge1_recall_stderr": 0.0025222065017164704, "rouge2_fmeasure": 0.09565357595328416, "rouge2_fmeasure_stderr": 0.0015669516035843776, "rouge2_precision": 0.11056115709540705, "rouge2_precision_stderr": 0.0020146080400812697, "rouge2_recall": 0.09523241025254957, "rouge2_recall_stderr": 0.0016332379953558076, "rougeL_fmeasure": 0.22884790375880742, "rougeL_fmeasure_stderr": 0.0017544092909052675, "rougeL_precision": 0.2614140651257123, "rougeL_precision_stderr": 0.002409323599550188, "rougeL_recall": 0.22758795096382134, "rougeL_recall_stderr": 0.00197916645130277, "rougeLsum_fmeasure": 0.26001996395899674, "rougeLsum_fmeasure_stderr": 0.002021020490775709, "rougeLsum_precision": 0.29587798141349847, "rougeLsum_precision_stderr": 0.0026666088968318393, "rougeLsum_recall": 0.2582635886545619, "rougeLsum_recall_stderr": 0.0022422350235981096}, "create_text_for_me": {"bleu": 5.319483620066844, "bleu_stderr": 0.1937918156309555, "rouge1_fmeasure": 0.3137620150662928, "rouge1_fmeasure_stderr": 0.0021267449025118832, "rouge1_precision": 0.3560891632971412, "rouge1_precision_stderr": 0.0028052847609035393, "rouge1_recall": 0.3118566460746229, "rouge1_recall_stderr": 0.0024563894577858706, "rouge2_fmeasure": 0.09313015798234085, "rouge2_fmeasure_stderr": 0.00152028521858163, "rouge2_precision": 0.10787055205478631, "rouge2_precision_stderr": 0.001961259785243488, "rouge2_recall": 0.09277219175388018, "rouge2_recall_stderr": 0.001599521459285887, "rougeL_fmeasure": 0.23055494373648347, "rougeL_fmeasure_stderr": 0.001676008041882916, "rougeL_precision": 0.2639125125532517, "rougeL_precision_stderr": 0.0023507470257026764, "rougeL_recall": 0.22918005986102416, "rougeL_recall_stderr": 0.001931758070692363, "rougeLsum_fmeasure": 0.2627254077501295, "rougeLsum_fmeasure_stderr": 0.0019364234716888252, "rougeLsum_precision": 0.2995336579782984, "rougeLsum_precision_stderr": 0.002598520129287253, "rougeLsum_recall": 0.2608326939117505, "rougeLsum_recall_stderr": 0.00218886717995965}, "generate_gramatically_correct_text": {"bleu": 4.017338372139203, "bleu_stderr": 0.11094350038266719, "rouge1_fmeasure": 0.24439453610800832, "rouge1_fmeasure_stderr": 0.002471192570902368, "rouge1_precision": 0.2695313480190455, "rouge1_precision_stderr": 0.0030112072928501336, "rouge1_recall": 0.24782182532367983, "rouge1_recall_stderr": 0.0027049982514322063, "rouge2_fmeasure": 0.07759858672584617, "rouge2_fmeasure_stderr": 0.0014022028282609771, "rouge2_precision": 0.08550020741401276, "rouge2_precision_stderr": 0.0016123330184419132, "rouge2_recall": 0.07891710085141755, "rouge2_recall_stderr": 0.0015002516695171015, "rougeL_fmeasure": 0.1891847721343501, "rougeL_fmeasure_stderr": 0.0017507687945178178, "rougeL_precision": 0.20874372759839854, "rougeL_precision_stderr": 0.00219378411400417, "rougeL_recall": 0.19212524503414735, "rougeL_recall_stderr": 0.0019738605195842055, "rougeLsum_fmeasure": 0.20519809770842534, "rougeLsum_fmeasure_stderr": 0.0021184619122727845, "rougeLsum_precision": 0.2269415936324815, "rougeLsum_precision_stderr": 0.00261469191061651, "rougeLsum_recall": 0.20793572411416514, "rougeLsum_recall_stderr": 0.0023144943247615053}, "generate_text_restaurant": {"bleu": 5.251573482634308, "bleu_stderr": 0.1553666643851091, "rouge1_fmeasure": 0.31100455847956715, "rouge1_fmeasure_stderr": 0.002052592278425154, "rouge1_precision": 0.36700582996097203, "rouge1_precision_stderr": 0.002866361764312095, "rouge1_recall": 0.29908671329451914, "rouge1_recall_stderr": 0.0022842305263675784, "rouge2_fmeasure": 0.09730358308616667, "rouge2_fmeasure_stderr": 0.001525935012941263, "rouge2_precision": 0.11715256391592369, "rouge2_precision_stderr": 0.002004164016012016, "rouge2_recall": 0.09355545687055947, "rouge2_recall_stderr": 0.0015451237546724788, "rougeL_fmeasure": 0.2352915422950905, "rougeL_fmeasure_stderr": 0.001672302120038204, "rougeL_precision": 0.2792795885938201, "rougeL_precision_stderr": 0.002414534192644447, "rougeL_recall": 0.22654007734086326, "rougeL_recall_stderr": 0.0018601465471659855, "rougeLsum_fmeasure": 0.26183889011003636, "rougeLsum_fmeasure_stderr": 0.001895806329444741, "rougeLsum_precision": 0.3101815290461886, "rougeLsum_precision_stderr": 0.0026513885447339244, "rougeLsum_recall": 0.25160776782553634, "rougeLsum_recall_stderr": 0.0020719183900535583}, "text": {"bleu": 5.326249606769673, "bleu_stderr": 0.13870766757762862, "rouge1_fmeasure": 0.31118667832544594, "rouge1_fmeasure_stderr": 0.0020471931429837612, "rouge1_precision": 0.3689871087097193, "rouge1_precision_stderr": 0.0029017082283349553, "rouge1_recall": 0.3001344146565885, "rouge1_recall_stderr": 0.0022905081784066715, "rouge2_fmeasure": 0.0981873842191505, "rouge2_fmeasure_stderr": 0.0015253172592372794, "rouge2_precision": 0.12022586215655875, "rouge2_precision_stderr": 0.0020979416142716933, "rouge2_recall": 0.09432946255117527, "rouge2_recall_stderr": 0.0015316319038071217, "rougeL_fmeasure": 0.23662330266166123, "rougeL_fmeasure_stderr": 0.001654720258750121, "rougeL_precision": 0.2833630391152307, "rougeL_precision_stderr": 0.0025014801176794616, "rougeL_recall": 0.2280006358827552, "rougeL_recall_stderr": 0.0018282950889621254, "rougeLsum_fmeasure": 0.2632319188900435, "rougeLsum_fmeasure_stderr": 0.001875582937147226, "rougeLsum_precision": 0.31356761141424766, "rougeLsum_precision_stderr": 0.002691111456370736, "rougeLsum_recall": 0.25378328346517437, "rougeLsum_recall_stderr": 0.002063376374536576}}}, "gem_xsum": {"0": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.47457826897610184, "bleu_stderr": 0.06622527858136913, "rouge1_fmeasure": 0.10930956321205572, "rouge1_fmeasure_stderr": 0.0018636365300419618, "rouge1_precision": 0.07849100625603515, "rouge1_precision_stderr": 0.0013763236409170894, "rouge1_recall": 0.18830692834903057, "rouge1_recall_stderr": 0.003201584532139998, "rouge2_fmeasure": 0.014331752023111665, "rouge2_fmeasure_stderr": 0.0008038622113235613, "rouge2_precision": 0.0101373697747153, "rouge2_precision_stderr": 0.0005725799485965975, "rouge2_recall": 0.02554990695060758, "rouge2_recall_stderr": 0.0014536715782248486, "rougeL_fmeasure": 0.09955783945841405, "rougeL_fmeasure_stderr": 0.0015927736901503137, "rougeL_precision": 0.07153361180580091, "rougeL_precision_stderr": 0.001183235553725598, "rougeL_recall": 0.171439117832265, "rougeL_recall_stderr": 0.0027508968911593348, "rougeLsum_fmeasure": 0.08714871620375533, "rougeLsum_fmeasure_stderr": 0.0014790422117723686, "rougeLsum_precision": 0.06237679092272722, "rougeLsum_precision_stderr": 0.0010796552208218652, "rougeLsum_recall": 0.1514930768097993, "rougeLsum_recall_stderr": 0.0026421209389933848}, "DOC_tldr": {"bleu": 0.5835098927919238, "bleu_stderr": 0.07719313039217032, "rouge1_fmeasure": 0.10741115659863423, "rouge1_fmeasure_stderr": 0.0020029906467731195, "rouge1_precision": 0.07695974919266163, "rouge1_precision_stderr": 0.0014773165827520049, "rouge1_recall": 0.18662101528755234, "rouge1_recall_stderr": 0.0034322790231989847, "rouge2_fmeasure": 0.015793527267809353, "rouge2_fmeasure_stderr": 0.0008540265285033742, "rouge2_precision": 0.01122477293725975, "rouge2_precision_stderr": 0.0006155567811512096, "rouge2_recall": 0.028017590709361054, "rouge2_recall_stderr": 0.0015078040578957851, "rougeL_fmeasure": 0.09765196595353162, "rougeL_fmeasure_stderr": 0.0017221112550801897, "rougeL_precision": 0.06992993909202184, "rougeL_precision_stderr": 0.0012721116443108543, "rougeL_recall": 0.16999666228834834, "rougeL_recall_stderr": 0.0029761829036210304, "rougeLsum_fmeasure": 0.08537020359960786, "rougeLsum_fmeasure_stderr": 0.0015426696504085384, "rougeLsum_precision": 0.061013847977830764, "rougeLsum_precision_stderr": 0.0011300630372763191, "rougeLsum_recall": 0.14954259216443205, "rougeLsum_recall_stderr": 0.0027335378592026204}, "article_DOC_summary": {"bleu": 0.6141824165767692, "bleu_stderr": 0.07244482599281088, "rouge1_fmeasure": 0.12137792760871402, "rouge1_fmeasure_stderr": 0.002184184971255157, "rouge1_precision": 0.08684854788612234, "rouge1_precision_stderr": 0.0016136017598306646, "rouge1_recall": 0.21068678976862762, "rouge1_recall_stderr": 0.0037297139710485957, "rouge2_fmeasure": 0.01783873426577586, "rouge2_fmeasure_stderr": 0.0009275277075872498, "rouge2_precision": 0.012636632651205262, "rouge2_precision_stderr": 0.0006618849333669708, "rouge2_recall": 0.0316888941525413, "rouge2_recall_stderr": 0.0016796888756788677, "rougeL_fmeasure": 0.10669020751320313, "rougeL_fmeasure_stderr": 0.0017737444033652681, "rougeL_precision": 0.07624380858300416, "rougeL_precision_stderr": 0.0013077167367173933, "rougeL_recall": 0.18587184963214815, "rougeL_recall_stderr": 0.0030939829912028922, "rougeLsum_fmeasure": 0.09494249315705383, "rougeLsum_fmeasure_stderr": 0.0016928172404188113, "rougeLsum_precision": 0.06773047218424798, "rougeLsum_precision_stderr": 0.0012398806155871941, "rougeLsum_recall": 0.16635584373488718, "rougeLsum_recall_stderr": 0.0030019144404050606}, "summarize_DOC": {"bleu": 0.5173175787874577, "bleu_stderr": 0.055369098931755845, "rouge1_fmeasure": 0.11615011773689751, "rouge1_fmeasure_stderr": 0.0020132702860050805, "rouge1_precision": 0.08294449475355638, "rouge1_precision_stderr": 0.001501101677358941, "rouge1_recall": 0.20318981866441785, "rouge1_recall_stderr": 0.003451770742452455, "rouge2_fmeasure": 0.01634264454041957, "rouge2_fmeasure_stderr": 0.0008732318540767917, "rouge2_precision": 0.0115654770629999, "rouge2_precision_stderr": 0.000620636009223327, "rouge2_recall": 0.029012948159446418, "rouge2_recall_stderr": 0.0015609513028227002, "rougeL_fmeasure": 0.10497029887714841, "rougeL_fmeasure_stderr": 0.0016965946158159747, "rougeL_precision": 0.07485979068030396, "rougeL_precision_stderr": 0.0012592589840183614, "rougeL_recall": 0.18415577121037147, "rougeL_recall_stderr": 0.0029429725468367554, "rougeLsum_fmeasure": 0.091932294749694, "rougeLsum_fmeasure_stderr": 0.0015639138169942146, "rougeLsum_precision": 0.0654649026019978, "rougeLsum_precision_stderr": 0.0011528108608884256, "rougeLsum_recall": 0.16188192696498585, "rougeLsum_recall_stderr": 0.0027459266075334754}, "summarize_this_DOC_summary": {"bleu": 0.6162431159385546, "bleu_stderr": 0.0719411252845223, "rouge1_fmeasure": 0.11344123075110753, "rouge1_fmeasure_stderr": 0.002075752711472246, "rouge1_precision": 0.08101405207575763, "rouge1_precision_stderr": 0.0015402709854789834, "rouge1_recall": 0.1978371464700832, "rouge1_recall_stderr": 0.0034969455098272624, "rouge2_fmeasure": 0.016637993147190867, "rouge2_fmeasure_stderr": 0.0009926063504469168, "rouge2_precision": 0.01181630825554818, "rouge2_precision_stderr": 0.0007223475673028841, "rouge2_recall": 0.029271962971207783, "rouge2_recall_stderr": 0.0016894106447306921, "rougeL_fmeasure": 0.10349719856532423, "rougeL_fmeasure_stderr": 0.00178021572550937, "rougeL_precision": 0.07385599916218046, "rougeL_precision_stderr": 0.0013240985175350677, "rougeL_recall": 0.1808620333503495, "rougeL_recall_stderr": 0.0030065019141283108, "rougeLsum_fmeasure": 0.08884476245308559, "rougeLsum_fmeasure_stderr": 0.0016345699043350426, "rougeLsum_precision": 0.06323384533522881, "rougeLsum_precision_stderr": 0.0012031428852023567, "rougeLsum_recall": 0.1564882734228425, "rougeLsum_recall_stderr": 0.0028531783790494997}}, "1": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.6132540544643678, "bleu_stderr": 0.0713885076138697, "rouge1_fmeasure": 0.13474307731979848, "rouge1_fmeasure_stderr": 0.002111986178677571, "rouge1_precision": 0.09609231578553688, "rouge1_precision_stderr": 0.0015668691591351292, "rouge1_recall": 0.23552842368238433, "rouge1_recall_stderr": 0.003577814966154032, "rouge2_fmeasure": 0.01563559121007456, "rouge2_fmeasure_stderr": 0.0009139886175241735, "rouge2_precision": 0.0110354532279438, "rouge2_precision_stderr": 0.0006463903956640263, "rouge2_recall": 0.027990471006616467, "rouge2_recall_stderr": 0.001658813041921148, "rougeL_fmeasure": 0.10311323732086543, "rougeL_fmeasure_stderr": 0.0015375375103156618, "rougeL_precision": 0.0734302137042163, "rougeL_precision_stderr": 0.001138075001700217, "rougeL_recall": 0.18134020183601798, "rougeL_recall_stderr": 0.0026912992930091363, "rougeLsum_fmeasure": 0.10726294947450196, "rougeLsum_fmeasure_stderr": 0.0016523566097021156, "rougeLsum_precision": 0.07630876213778368, "rougeLsum_precision_stderr": 0.0012122651784603749, "rougeLsum_recall": 0.18889813705308528, "rougeLsum_recall_stderr": 0.0029100234368416634}, "DOC_tldr": {"bleu": 0.5393663863983892, "bleu_stderr": 0.05348433562375772, "rouge1_fmeasure": 0.12292814399550107, "rouge1_fmeasure_stderr": 0.002063269026074902, "rouge1_precision": 0.0998197720955355, "rouge1_precision_stderr": 0.001791958979970089, "rouge1_recall": 0.19150976792164776, "rouge1_recall_stderr": 0.003597233562430454, "rouge2_fmeasure": 0.012260258344477375, "rouge2_fmeasure_stderr": 0.0008029144977571671, "rouge2_precision": 0.00926872748126837, "rouge2_precision_stderr": 0.0006094283518785105, "rouge2_recall": 0.020793570369667533, "rouge2_recall_stderr": 0.0014099011327662694, "rougeL_fmeasure": 0.09847662810007793, "rougeL_fmeasure_stderr": 0.0015558437843189248, "rougeL_precision": 0.08018556273037858, "rougeL_precision_stderr": 0.0013905565313662618, "rougeL_recall": 0.1534507156318649, "rougeL_recall_stderr": 0.002735628237761709, "rougeLsum_fmeasure": 0.09851599027450329, "rougeLsum_fmeasure_stderr": 0.001607691573132454, "rougeLsum_precision": 0.08017202493423412, "rougeLsum_precision_stderr": 0.0014158245846160884, "rougeLsum_recall": 0.1538977801609464, "rougeLsum_recall_stderr": 0.00287738921463875}, "article_DOC_summary": {"bleu": 0.8152552954236955, "bleu_stderr": 0.1031507916283277, "rouge1_fmeasure": 0.13729478048942775, "rouge1_fmeasure_stderr": 0.0024098935839123887, "rouge1_precision": 0.09856438801955326, "rouge1_precision_stderr": 0.0017913961749217754, "rouge1_recall": 0.23807395731457542, "rouge1_recall_stderr": 0.00408145979752075, "rouge2_fmeasure": 0.02086687296490721, "rouge2_fmeasure_stderr": 0.0011212733091223604, "rouge2_precision": 0.014805002932563531, "rouge2_precision_stderr": 0.0008137316866053345, "rouge2_recall": 0.03704847337143438, "rouge2_recall_stderr": 0.0019513225466723323, "rougeL_fmeasure": 0.11005303698684504, "rougeL_fmeasure_stderr": 0.0017730014084980385, "rougeL_precision": 0.07886562535884908, "rougeL_precision_stderr": 0.001319440885298338, "rougeL_recall": 0.19199210330518746, "rougeL_recall_stderr": 0.0030607720042053164, "rougeLsum_fmeasure": 0.10790506830750389, "rougeLsum_fmeasure_stderr": 0.0019575871972268514, "rougeLsum_precision": 0.07731479241226244, "rougeLsum_precision_stderr": 0.0014429529345158431, "rougeLsum_recall": 0.18826403944987077, "rougeLsum_recall_stderr": 0.0033888132406159205}, "summarize_DOC": {"bleu": 0.617354720838304, "bleu_stderr": 0.06307890231427365, "rouge1_fmeasure": 0.12132677528862063, "rouge1_fmeasure_stderr": 0.0021172474639601026, "rouge1_precision": 0.08748874292219283, "rouge1_precision_stderr": 0.0016046785625036608, "rouge1_recall": 0.21040419098884006, "rouge1_recall_stderr": 0.003596032405414458, "rouge2_fmeasure": 0.017118068362416517, "rouge2_fmeasure_stderr": 0.00091598354274057, "rouge2_precision": 0.012177718487699685, "rouge2_precision_stderr": 0.0006600449164619634, "rouge2_recall": 0.03041617462460392, "rouge2_recall_stderr": 0.0016279180469739754, "rougeL_fmeasure": 0.10363956196342357, "rougeL_fmeasure_stderr": 0.0016890572561683093, "rougeL_precision": 0.07463066246768363, "rougeL_precision_stderr": 0.0012825612167355926, "rougeL_recall": 0.18070029655502712, "rougeL_recall_stderr": 0.0029520412330890744, "rougeLsum_fmeasure": 0.0961985987954344, "rougeLsum_fmeasure_stderr": 0.0016792470214151791, "rougeLsum_precision": 0.06928118345477047, "rougeLsum_precision_stderr": 0.0012737323592854019, "rougeLsum_recall": 0.16792413773506376, "rougeLsum_recall_stderr": 0.0029340056636205994}, "summarize_this_DOC_summary": {"bleu": 0.6233103019291443, "bleu_stderr": 0.10322680839865134, "rouge1_fmeasure": 0.12537834249633983, "rouge1_fmeasure_stderr": 0.0022469691506953465, "rouge1_precision": 0.09212077005868066, "rouge1_precision_stderr": 0.0017630634952107368, "rouge1_recall": 0.21369522061211693, "rouge1_recall_stderr": 0.003729436274385912, "rouge2_fmeasure": 0.01660477741264796, "rouge2_fmeasure_stderr": 0.0010056200276438794, "rouge2_precision": 0.01206376214517874, "rouge2_precision_stderr": 0.0007516662252398531, "rouge2_recall": 0.02903754360252403, "rouge2_recall_stderr": 0.0017031825569670093, "rougeL_fmeasure": 0.10446236594583992, "rougeL_fmeasure_stderr": 0.0017570999718229995, "rougeL_precision": 0.07655663031331215, "rougeL_precision_stderr": 0.0013855875426255315, "rougeL_recall": 0.1792751829195347, "rougeL_recall_stderr": 0.0029860001198346964, "rougeLsum_fmeasure": 0.09853403043110821, "rougeLsum_fmeasure_stderr": 0.0017820950507240858, "rougeLsum_precision": 0.07232336786129806, "rougeLsum_precision_stderr": 0.00140309509381785, "rougeLsum_recall": 0.1689005605679794, "rougeLsum_recall_stderr": 0.0030087684057466145}}, "2": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.6167432807079934, "bleu_stderr": 0.07313238207352012, "rouge1_fmeasure": 0.1313944002337063, "rouge1_fmeasure_stderr": 0.001990658173774306, "rouge1_precision": 0.09383796110256946, "rouge1_precision_stderr": 0.0014865897795411922, "rouge1_recall": 0.2300444944691374, "rouge1_recall_stderr": 0.003392043854938688, "rouge2_fmeasure": 0.01551951461927907, "rouge2_fmeasure_stderr": 0.0008825861261678188, "rouge2_precision": 0.01098830723574462, "rouge2_precision_stderr": 0.0006303776417170931, "rouge2_recall": 0.027489027406269212, "rouge2_recall_stderr": 0.0015570004325674606, "rougeL_fmeasure": 0.10329061631237338, "rougeL_fmeasure_stderr": 0.0014671878058074702, "rougeL_precision": 0.07370182535719132, "rougeL_precision_stderr": 0.001100110897417761, "rougeL_recall": 0.1817594002432442, "rougeL_recall_stderr": 0.002569209339964884, "rougeLsum_fmeasure": 0.10446611375407802, "rougeLsum_fmeasure_stderr": 0.0016139685037892917, "rougeLsum_precision": 0.07447014880573084, "rougeLsum_precision_stderr": 0.0011974801492887503, "rougeLsum_recall": 0.18408290764630678, "rougeLsum_recall_stderr": 0.002832630645866158}, "DOC_tldr": {"bleu": 0.4248411619556655, "bleu_stderr": 0.04337063842371632, "rouge1_fmeasure": 0.12080327979692661, "rouge1_fmeasure_stderr": 0.0019325844096013765, "rouge1_precision": 0.09588515299897513, "rouge1_precision_stderr": 0.0018389413668098858, "rouge1_recall": 0.1941351180627413, "rouge1_recall_stderr": 0.0032321812427151307, "rouge2_fmeasure": 0.012201390939302973, "rouge2_fmeasure_stderr": 0.0007193995074256504, "rouge2_precision": 0.00963799324260621, "rouge2_precision_stderr": 0.0006615867592044398, "rouge2_recall": 0.020245217917730207, "rouge2_recall_stderr": 0.0011797301011867512, "rougeL_fmeasure": 0.10138824592737848, "rougeL_fmeasure_stderr": 0.0015778531626384088, "rougeL_precision": 0.08057175111690566, "rougeL_precision_stderr": 0.0015714221610920258, "rougeL_recall": 0.1632996891477624, "rougeL_recall_stderr": 0.0026283320276119515, "rougeLsum_fmeasure": 0.09694968298249196, "rougeLsum_fmeasure_stderr": 0.0015608436285872904, "rougeLsum_precision": 0.07734192765900497, "rougeLsum_precision_stderr": 0.0015709846641160572, "rougeLsum_recall": 0.1560636734165814, "rougeLsum_recall_stderr": 0.002615177895498355}, "article_DOC_summary": {"bleu": 0.4783982759526829, "bleu_stderr": 0.041614834308327325, "rouge1_fmeasure": 0.12396034301655152, "rouge1_fmeasure_stderr": 0.0021408486311541627, "rouge1_precision": 0.08892601657513456, "rouge1_precision_stderr": 0.0016077156412850602, "rouge1_recall": 0.21529785402215762, "rouge1_recall_stderr": 0.0035920176085112838, "rouge2_fmeasure": 0.015790474040565182, "rouge2_fmeasure_stderr": 0.000831397853071339, "rouge2_precision": 0.011265761012393422, "rouge2_precision_stderr": 0.0006031851124179686, "rouge2_recall": 0.0276901962071342, "rouge2_recall_stderr": 0.0014341330838498428, "rougeL_fmeasure": 0.10371163259608823, "rougeL_fmeasure_stderr": 0.0016126497685603005, "rougeL_precision": 0.07425540521394834, "rougeL_precision_stderr": 0.00121183146821238, "rougeL_recall": 0.18108047541087044, "rougeL_recall_stderr": 0.0027630960930183058, "rougeLsum_fmeasure": 0.09738815767936584, "rougeLsum_fmeasure_stderr": 0.0017074564850241471, "rougeLsum_precision": 0.06968809353220383, "rougeLsum_precision_stderr": 0.0012730861197946873, "rougeLsum_recall": 0.17037941440049567, "rougeLsum_recall_stderr": 0.0029297652528551157}, "summarize_DOC": {"bleu": 0.528150461312332, "bleu_stderr": 0.04887622861962553, "rouge1_fmeasure": 0.11881862682601178, "rouge1_fmeasure_stderr": 0.0019599355384508203, "rouge1_precision": 0.08502500988635123, "rouge1_precision_stderr": 0.001498937813320831, "rouge1_recall": 0.2084492768665317, "rouge1_recall_stderr": 0.0033474975083069305, "rouge2_fmeasure": 0.015337369323206902, "rouge2_fmeasure_stderr": 0.0008247870081098087, "rouge2_precision": 0.010957922059575794, "rouge2_precision_stderr": 0.0006364547528005359, "rouge2_recall": 0.02745994543864676, "rouge2_recall_stderr": 0.0014679599804070377, "rougeL_fmeasure": 0.10352955674946399, "rougeL_fmeasure_stderr": 0.0016013916021151678, "rougeL_precision": 0.0739732762619997, "rougeL_precision_stderr": 0.0012237819610442023, "rougeL_recall": 0.18227843033716665, "rougeL_recall_stderr": 0.0027816909692955204, "rougeLsum_fmeasure": 0.09449014264304802, "rougeLsum_fmeasure_stderr": 0.001581094844660653, "rougeLsum_precision": 0.06740414152435432, "rougeLsum_precision_stderr": 0.0011978138296961046, "rougeLsum_recall": 0.16720075001064724, "rougeLsum_recall_stderr": 0.002795799505244066}, "summarize_this_DOC_summary": {"bleu": 0.4996183843525882, "bleu_stderr": 0.04001578639941148, "rouge1_fmeasure": 0.11974042458990046, "rouge1_fmeasure_stderr": 0.002030784483754028, "rouge1_precision": 0.08661269663304301, "rouge1_precision_stderr": 0.001593838459943478, "rouge1_recall": 0.20717931900250192, "rouge1_recall_stderr": 0.0033843575265547453, "rouge2_fmeasure": 0.015360819588816904, "rouge2_fmeasure_stderr": 0.000809551101724943, "rouge2_precision": 0.011007610065072464, "rouge2_precision_stderr": 0.0005989306753914806, "rouge2_recall": 0.027125371681868536, "rouge2_recall_stderr": 0.0014187760277359451, "rougeL_fmeasure": 0.10256512747901998, "rougeL_fmeasure_stderr": 0.001605784979744158, "rougeL_precision": 0.07395213433806418, "rougeL_precision_stderr": 0.0012575784598958418, "rougeL_recall": 0.17867696254495863, "rougeL_recall_stderr": 0.002758667555218375, "rougeLsum_fmeasure": 0.09286761848139825, "rougeLsum_fmeasure_stderr": 0.001559907403116052, "rougeLsum_precision": 0.06701000937643628, "rougeLsum_precision_stderr": 0.0012330207864759619, "rougeLsum_recall": 0.16211217586867194, "rougeLsum_recall_stderr": 0.0026761095348479098}}, "3": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.6537439527112324, "bleu_stderr": 0.08789003420216455, "rouge1_fmeasure": 0.12223241280960181, "rouge1_fmeasure_stderr": 0.0022597737898330176, "rouge1_precision": 0.08929232207008256, "rouge1_precision_stderr": 0.0017661136394464746, "rouge1_recall": 0.2083714353056586, "rouge1_recall_stderr": 0.0037507822542893463, "rouge2_fmeasure": 0.014615300016460326, "rouge2_fmeasure_stderr": 0.0009436727425446628, "rouge2_precision": 0.010779714787610186, "rouge2_precision_stderr": 0.0007228389047199584, "rouge2_recall": 0.024684029325778328, "rouge2_recall_stderr": 0.0015436289252720226, "rougeL_fmeasure": 0.09843060072407699, "rougeL_fmeasure_stderr": 0.0017199014486665768, "rougeL_precision": 0.07198363558556994, "rougeL_precision_stderr": 0.0013768675700732372, "rougeL_recall": 0.168224053314788, "rougeL_recall_stderr": 0.002856247526588458, "rougeLsum_fmeasure": 0.097565135698147, "rougeLsum_fmeasure_stderr": 0.001816709165840859, "rougeLsum_precision": 0.07122439300395828, "rougeLsum_precision_stderr": 0.0014339663669546733, "rougeLsum_recall": 0.16714051967493118, "rougeLsum_recall_stderr": 0.00304885339797809}, "DOC_tldr": {"bleu": 0.5384672086268061, "bleu_stderr": 0.05628261735256145, "rouge1_fmeasure": 0.11838286391586506, "rouge1_fmeasure_stderr": 0.0022575984998313976, "rouge1_precision": 0.0979524500570362, "rouge1_precision_stderr": 0.0023574213995995338, "rouge1_recall": 0.18432267940454714, "rouge1_recall_stderr": 0.003449394808563609, "rouge2_fmeasure": 0.014849672103797189, "rouge2_fmeasure_stderr": 0.0009398723971147706, "rouge2_precision": 0.01223431053167332, "rouge2_precision_stderr": 0.0009139707393864935, "rouge2_recall": 0.023333668667089247, "rouge2_recall_stderr": 0.001381718229050345, "rougeL_fmeasure": 0.1015835231933467, "rougeL_fmeasure_stderr": 0.0018425036048339849, "rougeL_precision": 0.08382542142444173, "rougeL_precision_stderr": 0.001948560716573151, "rougeL_recall": 0.15921404653049828, "rougeL_recall_stderr": 0.002868932069201499, "rougeLsum_fmeasure": 0.09581512027677436, "rougeLsum_fmeasure_stderr": 0.0018225664745385676, "rougeLsum_precision": 0.0796692729044991, "rougeLsum_precision_stderr": 0.001955663224730415, "rougeLsum_recall": 0.14966156017894405, "rougeLsum_recall_stderr": 0.0028188651661202858}, "article_DOC_summary": {"bleu": 0.5544562884765827, "bleu_stderr": 0.03868981500279069, "rouge1_fmeasure": 0.11451534571985687, "rouge1_fmeasure_stderr": 0.002245243404997308, "rouge1_precision": 0.08435225307357676, "rouge1_precision_stderr": 0.0017949929630033349, "rouge1_recall": 0.19524716118009153, "rouge1_recall_stderr": 0.003729643037639876, "rouge2_fmeasure": 0.015065187296172565, "rouge2_fmeasure_stderr": 0.0008474739343157203, "rouge2_precision": 0.010931147147210569, "rouge2_precision_stderr": 0.000631905230395927, "rouge2_recall": 0.026021044054339443, "rouge2_recall_stderr": 0.0014337475560734076, "rougeL_fmeasure": 0.09755266669239072, "rougeL_fmeasure_stderr": 0.0017557981664671374, "rougeL_precision": 0.07171273980778682, "rougeL_precision_stderr": 0.0014130706113843967, "rougeL_recall": 0.167240338703402, "rougeL_recall_stderr": 0.0029869789558385, "rougeLsum_fmeasure": 0.09051613592037677, "rougeLsum_fmeasure_stderr": 0.0017384555593769912, "rougeLsum_precision": 0.06667665265151085, "rougeLsum_precision_stderr": 0.0014209048598926293, "rougeLsum_recall": 0.15545938043826846, "rougeLsum_recall_stderr": 0.002970962161003125}, "summarize_DOC": {"bleu": 0.4932998587527608, "bleu_stderr": 0.04917973267537893, "rouge1_fmeasure": 0.11303815097050755, "rouge1_fmeasure_stderr": 0.002085394908569434, "rouge1_precision": 0.08286382508828975, "rouge1_precision_stderr": 0.00171211565369596, "rouge1_recall": 0.19417127366399595, "rouge1_recall_stderr": 0.0035347086651873116, "rouge2_fmeasure": 0.014667451968252554, "rouge2_fmeasure_stderr": 0.0008103234054454378, "rouge2_precision": 0.010709281907276956, "rouge2_precision_stderr": 0.0006463393919380488, "rouge2_recall": 0.02563055239095909, "rouge2_recall_stderr": 0.0013753043663831622, "rougeL_fmeasure": 0.09884209508770671, "rougeL_fmeasure_stderr": 0.0017207139349791946, "rougeL_precision": 0.0723252465078549, "rougeL_precision_stderr": 0.0014008899550456907, "rougeL_recall": 0.17025301778061233, "rougeL_recall_stderr": 0.002944125742781433, "rougeLsum_fmeasure": 0.0888642550425071, "rougeLsum_fmeasure_stderr": 0.0016417562882641062, "rougeLsum_precision": 0.0650622060288472, "rougeLsum_precision_stderr": 0.0013556680905752983, "rougeLsum_recall": 0.1537510648011344, "rougeLsum_recall_stderr": 0.0028634633132608622}, "summarize_this_DOC_summary": {"bleu": 0.5195966300461277, "bleu_stderr": 0.042459426003985785, "rouge1_fmeasure": 0.11040951377527043, "rouge1_fmeasure_stderr": 0.002179873874618349, "rouge1_precision": 0.0806895748479458, "rouge1_precision_stderr": 0.0017148726001275283, "rouge1_recall": 0.18907123674289394, "rouge1_recall_stderr": 0.0036401681772858063, "rouge2_fmeasure": 0.014649332777625616, "rouge2_fmeasure_stderr": 0.0008477225467246834, "rouge2_precision": 0.01053868364581689, "rouge2_precision_stderr": 0.0006210803937322794, "rouge2_recall": 0.025668261986885543, "rouge2_recall_stderr": 0.0014569341217374899, "rougeL_fmeasure": 0.09609527909787907, "rougeL_fmeasure_stderr": 0.001772609518829833, "rougeL_precision": 0.07010786798662083, "rougeL_precision_stderr": 0.0013932486949560677, "rougeL_recall": 0.16531909956669813, "rougeL_recall_stderr": 0.0030227929410000416, "rougeLsum_fmeasure": 0.08755892275631494, "rougeLsum_fmeasure_stderr": 0.0016919775746021333, "rougeLsum_precision": 0.06390479614025571, "rougeLsum_precision_stderr": 0.0013316010773909046, "rougeLsum_recall": 0.15100544107920313, "rougeLsum_recall_stderr": 0.0028997429382766494}}, "4": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.38568323827085005, "bleu_stderr": 0.13140258824575432, "rouge1_fmeasure": 0.030623951982172243, "rouge1_fmeasure_stderr": 0.0019723679586947157, "rouge1_precision": 0.027438753007734348, "rouge1_precision_stderr": 0.0020209855248670377, "rouge1_recall": 0.047135286894855855, "rouge1_recall_stderr": 0.003093201466426986, "rouge2_fmeasure": 0.004331423800367673, "rouge2_fmeasure_stderr": 0.0006498030956306457, "rouge2_precision": 0.0036391643178718376, "rouge2_precision_stderr": 0.000651422116785187, "rouge2_recall": 0.007018854013603984, "rouge2_recall_stderr": 0.00101986519717702, "rougeL_fmeasure": 0.02559588917107183, "rougeL_fmeasure_stderr": 0.0016311116927956652, "rougeL_precision": 0.022758262766782358, "rougeL_precision_stderr": 0.0016467261403303932, "rougeL_recall": 0.03973720943624368, "rougeL_recall_stderr": 0.002596893758975351, "rougeLsum_fmeasure": 0.025026908052136167, "rougeLsum_fmeasure_stderr": 0.001623239492581538, "rougeLsum_precision": 0.02236476517938642, "rougeLsum_precision_stderr": 0.001641725788842373, "rougeLsum_recall": 0.03877314043404623, "rougeLsum_recall_stderr": 0.0025919403907963476}, "DOC_tldr": {"bleu": 0.30242530199415063, "bleu_stderr": 0.13172883284690357, "rouge1_fmeasure": 0.03268274482045124, "rouge1_fmeasure_stderr": 0.0020159690172151102, "rouge1_precision": 0.03078887567599419, "rouge1_precision_stderr": 0.002267198013309019, "rouge1_recall": 0.04728638198757095, "rouge1_recall_stderr": 0.0029563776305562995, "rouge2_fmeasure": 0.004600261742688949, "rouge2_fmeasure_stderr": 0.0006815109118145586, "rouge2_precision": 0.004097645217219916, "rouge2_precision_stderr": 0.0006352569090777523, "rouge2_recall": 0.0070103107833449875, "rouge2_recall_stderr": 0.0010527888125515437, "rougeL_fmeasure": 0.027882226495475622, "rougeL_fmeasure_stderr": 0.0017012154601545335, "rougeL_precision": 0.02580787661904382, "rougeL_precision_stderr": 0.0017950648224843338, "rougeL_recall": 0.04086841692598574, "rougeL_recall_stderr": 0.0025615977979740897, "rougeLsum_fmeasure": 0.02702176662269746, "rougeLsum_fmeasure_stderr": 0.0016712710261589593, "rougeLsum_precision": 0.025250388596127913, "rougeLsum_precision_stderr": 0.0017916684649173303, "rougeLsum_recall": 0.0392238241311461, "rougeLsum_recall_stderr": 0.0024646800606543846}, "article_DOC_summary": {"bleu": 0.23195231943321903, "bleu_stderr": 0.054461911286189185, "rouge1_fmeasure": 0.033226658072660585, "rouge1_fmeasure_stderr": 0.002003945138254869, "rouge1_precision": 0.028606404294185678, "rouge1_precision_stderr": 0.0020371981833308224, "rouge1_recall": 0.05177166241618001, "rouge1_recall_stderr": 0.003118570625917115, "rouge2_fmeasure": 0.004530040031767069, "rouge2_fmeasure_stderr": 0.0005161711108430996, "rouge2_precision": 0.004053086557676976, "rouge2_precision_stderr": 0.000644383420856288, "rouge2_recall": 0.007133537400455768, "rouge2_recall_stderr": 0.0007785336270850277, "rougeL_fmeasure": 0.028498538822695166, "rougeL_fmeasure_stderr": 0.0016689936239707673, "rougeL_precision": 0.02472463147760117, "rougeL_precision_stderr": 0.0017881996232063642, "rougeL_recall": 0.04459955593168214, "rougeL_recall_stderr": 0.0026285803260540815, "rougeLsum_fmeasure": 0.026611692302705815, "rougeLsum_fmeasure_stderr": 0.0016132169063633166, "rougeLsum_precision": 0.023409670499661615, "rougeLsum_precision_stderr": 0.0017830228948821282, "rougeLsum_recall": 0.041408985247042875, "rougeLsum_recall_stderr": 0.0024949239488166234}, "summarize_DOC": {"bleu": 0.3152874467334013, "bleu_stderr": 0.07390349309399162, "rouge1_fmeasure": 0.032765818131531135, "rouge1_fmeasure_stderr": 0.0019956975528504003, "rouge1_precision": 0.0299789202555777, "rouge1_precision_stderr": 0.0023310463175663133, "rouge1_recall": 0.05033299021370721, "rouge1_recall_stderr": 0.0030688602223686704, "rouge2_fmeasure": 0.004765891573358166, "rouge2_fmeasure_stderr": 0.0005748770709346177, "rouge2_precision": 0.003955464420202306, "rouge2_precision_stderr": 0.0005645114147565559, "rouge2_recall": 0.007608591554539119, "rouge2_recall_stderr": 0.0008849021378583971, "rougeL_fmeasure": 0.028427633970046932, "rougeL_fmeasure_stderr": 0.001695433517325568, "rougeL_precision": 0.02585211610210547, "rougeL_precision_stderr": 0.0019823318353263796, "rougeL_recall": 0.04414951473810684, "rougeL_recall_stderr": 0.0026739955507568457, "rougeLsum_fmeasure": 0.026135699640989368, "rougeLsum_fmeasure_stderr": 0.0015948963052667224, "rougeLsum_precision": 0.024288617878882954, "rougeLsum_precision_stderr": 0.0019683052842310715, "rougeLsum_recall": 0.04024052293200098, "rougeLsum_recall_stderr": 0.0024657230773210245}, "summarize_this_DOC_summary": {"bleu": 0.21210378195277477, "bleu_stderr": 0.05267300095484724, "rouge1_fmeasure": 0.029279070120697838, "rouge1_fmeasure_stderr": 0.001878769664661074, "rouge1_precision": 0.02658254011537247, "rouge1_precision_stderr": 0.0020748261577405607, "rouge1_recall": 0.04377731709316155, "rouge1_recall_stderr": 0.002784901327388771, "rouge2_fmeasure": 0.004119906223301282, "rouge2_fmeasure_stderr": 0.0005578664016300015, "rouge2_precision": 0.0044766254520966255, "rouge2_precision_stderr": 0.0010428981904857266, "rouge2_recall": 0.005945068157009263, "rouge2_recall_stderr": 0.0007338081846565158, "rougeL_fmeasure": 0.02506826654645877, "rougeL_fmeasure_stderr": 0.0015851273432252462, "rougeL_precision": 0.022618399699937414, "rougeL_precision_stderr": 0.0017607024402004165, "rougeL_recall": 0.03769542551971095, "rougeL_recall_stderr": 0.002353427232761915, "rougeLsum_fmeasure": 0.023443260455568726, "rougeLsum_fmeasure_stderr": 0.0015176085253178068, "rougeLsum_precision": 0.021501210074057614, "rougeLsum_precision_stderr": 0.0017408075966930808, "rougeLsum_recall": 0.034957744191750476, "rougeLsum_recall_stderr": 0.002217068454307888}}, "5": {"DOC_boils_down_to_simple_idea_that": {"bleu": 0.0, "bleu_stderr": 0.0, "rouge1_fmeasure": 0.0003741452934927028, "rouge1_fmeasure_stderr": 0.00017038377957226348, "rouge1_precision": 0.004288164665523156, "rouge1_precision_stderr": 0.0019144304746682786, "rouge1_recall": 0.00019598446840121887, "rouge1_recall_stderr": 8.942783707027923e-05, "rouge2_fmeasure": 0.0, "rouge2_fmeasure_stderr": 0.0, "rouge2_precision": 0.0, "rouge2_precision_stderr": 0.0, "rouge2_recall": 0.0, "rouge2_recall_stderr": 0.0, "rougeL_fmeasure": 0.0003741452934927028, "rougeL_fmeasure_stderr": 0.00017038377957226348, "rougeL_precision": 0.004288164665523156, "rougeL_precision_stderr": 0.0019144304746682786, "rougeL_recall": 0.00019598446840121887, "rougeL_recall_stderr": 8.942783707027923e-05, "rougeLsum_fmeasure": 0.0003741452934927028, "rougeLsum_fmeasure_stderr": 0.00017038377957226348, "rougeLsum_precision": 0.004288164665523156, "rougeLsum_precision_stderr": 0.0019144304746682786, "rougeLsum_recall": 0.00019598446840121887, "rougeLsum_recall_stderr": 8.942783707027923e-05}, "DOC_tldr": {"bleu": 7.569559051730558e-44, "bleu_stderr": 2.3751737130499017e-36, "rouge1_fmeasure": 0.0021779269831245112, "rouge1_fmeasure_stderr": 0.000636647151495716, "rouge1_precision": 0.0024946317337081986, "rouge1_precision_stderr": 0.0007459407337053418, "rouge1_recall": 0.0020432834635293762, "rouge1_recall_stderr": 0.0006122723724231832, "rouge2_fmeasure": 0.0001270035372229107, "rouge2_fmeasure_stderr": 7.339490340552063e-05, "rouge2_precision": 0.00015842386125404991, "rouge2_precision_stderr": 9.164096613628691e-05, "rouge2_recall": 0.0001078167115902965, "rouge2_recall_stderr": 6.282744363562356e-05, "rougeL_fmeasure": 0.001888812253805427, "rougeL_fmeasure_stderr": 0.0005328781795731039, "rougeL_precision": 0.002177974195907487, "rougeL_precision_stderr": 0.0006242303095573485, "rougeL_recall": 0.0017464607065091375, "rougeL_recall_stderr": 0.000498369445585121, "rougeLsum_fmeasure": 0.0019602816648974794, "rougeLsum_fmeasure_stderr": 0.0005561333735819992, "rougeLsum_precision": 0.002231576254226526, "rougeLsum_precision_stderr": 0.0006354831460676118, "rougeLsum_recall": 0.0018536648231472165, "rougeLsum_recall_stderr": 0.0005527691709280919}, "article_DOC_summary": {"bleu": 2.4598586849321513e-37, "bleu_stderr": 4.3357176079980746e-32, "rouge1_fmeasure": 0.002279670952470671, "rouge1_fmeasure_stderr": 0.0005995355838360033, "rouge1_precision": 0.002544888964583456, "rouge1_precision_stderr": 0.0006804984372875271, "rouge1_recall": 0.002139575841226647, "rouge1_recall_stderr": 0.0005562502135583049, "rouge2_fmeasure": 0.00021112542779058987, "rouge2_fmeasure_stderr": 0.00012666146238130252, "rouge2_precision": 0.000239072186907925, "rouge2_precision_stderr": 0.00014515151101602653, "rouge2_recall": 0.00019053273770254902, "rouge2_recall_stderr": 0.00011332476106214262, "rougeL_fmeasure": 0.0019409469526661608, "rougeL_fmeasure_stderr": 0.0005073885489964555, "rougeL_precision": 0.002161847481010399, "rougeL_precision_stderr": 0.0005749806010887603, "rougeL_recall": 0.0018323415576938233, "rougeL_recall_stderr": 0.0004753823971115864, "rougeLsum_fmeasure": 0.0020687632617786135, "rougeLsum_fmeasure_stderr": 0.0005471176517107834, "rougeLsum_precision": 0.0022997709059248868, "rougeLsum_precision_stderr": 0.0006186158865403682, "rougeLsum_recall": 0.0019535380516075003, "rougeLsum_recall_stderr": 0.0005119627013983172}, "summarize_DOC": {"bleu": 8.230901975344253e-37, "bleu_stderr": 1.6818144591792003e-31, "rouge1_fmeasure": 0.002850102432403856, "rouge1_fmeasure_stderr": 0.0007630282877736268, "rouge1_precision": 0.003145834181525519, "rouge1_precision_stderr": 0.0008641359171585344, "rouge1_recall": 0.0027491540556537565, "rouge1_recall_stderr": 0.0007447118682849743, "rouge2_fmeasure": 0.000538565784940234, "rouge2_fmeasure_stderr": 0.0002527457661965069, "rouge2_precision": 0.0006187008822968868, "rouge2_precision_stderr": 0.00029194397834022607, "rouge2_recall": 0.0004933724831937344, "rouge2_recall_stderr": 0.00023534159735572293, "rougeL_fmeasure": 0.00211639976529527, "rougeL_fmeasure_stderr": 0.0005548761683122237, "rougeL_precision": 0.0023242531040883747, "rougeL_precision_stderr": 0.0006266425117623151, "rougeL_recall": 0.0020438981820361424, "rougeL_recall_stderr": 0.0005399846248395976, "rougeLsum_fmeasure": 0.0024026672428148917, "rougeLsum_fmeasure_stderr": 0.0006455426387014536, "rougeLsum_precision": 0.002621636659839826, "rougeLsum_precision_stderr": 0.0007128110133984216, "rougeLsum_recall": 0.0023514083301448595, "rougeLsum_recall_stderr": 0.0006536022122755625}, "summarize_this_DOC_summary": {"bleu": 0.0, "bleu_stderr": 0.0, "rouge1_fmeasure": 0.0011581184102891899, "rouge1_fmeasure_stderr": 0.0003946303405826163, "rouge1_precision": 0.005288736420811892, "rouge1_precision_stderr": 0.0018191165622398336, "rouge1_recall": 0.0006712474238977358, "rouge1_recall_stderr": 0.000231168077080196, "rouge2_fmeasure": 0.00013885485583598793, "rouge2_fmeasure_stderr": 9.966211058781627e-05, "rouge2_precision": 0.0008576329331046312, "rouge2_precision_stderr": 0.0006061777330437759, "rouge2_recall": 7.576832303743922e-05, "rouge2_recall_stderr": 5.452796435684482e-05, "rougeL_fmeasure": 0.001083541633497483, "rougeL_fmeasure_stderr": 0.00035783272085016286, "rougeL_precision": 0.005002858776443681, "rougeL_precision_stderr": 0.0017037398901424319, "rougeL_recall": 0.0006283657772425043, "rougeL_recall_stderr": 0.00021043869960245262, "rougeLsum_fmeasure": 0.001083541633497483, "rougeLsum_fmeasure_stderr": 0.00035783272085016286, "rougeLsum_precision": 0.005002858776443681, "rougeLsum_precision_stderr": 0.0017037398901424319, "rougeLsum_recall": 0.0006283657772425043, "rougeLsum_recall_stderr": 0.00021043869960245262}}}, "piqa": {"0": {"Correct the solution": {"bleu": 3.656755366303797, "bleu_stderr": 0.14837890695680994, "rouge1_fmeasure": 0.09106715079841605, "rouge1_fmeasure_stderr": 0.002391471928167312, "rouge1_precision": 0.05669470574943534, "rouge1_precision_stderr": 0.0020170312039152695, "rouge1_recall": 0.5066296296346544, "rouge1_recall_stderr": 0.007659513774926616, "rouge2_fmeasure": 0.06256105007088962, "rouge2_fmeasure_stderr": 0.0022953455816733365, "rouge2_precision": 0.03860274611467539, "rouge2_precision_stderr": 0.00172983035557276, "rouge2_recall": 0.34854187631620626, "rouge2_recall_stderr": 0.00814423352083145, "rougeL_fmeasure": 0.08960277978128786, "rougeL_fmeasure_stderr": 0.0023665012918680627, "rougeL_precision": 0.05563894642256185, "rougeL_precision_stderr": 0.001982960805433701, "rougeL_recall": 0.5011913179283217, "rougeL_recall_stderr": 0.007700971199463536, "rougeLsum_fmeasure": 0.08521160683074443, "rougeLsum_fmeasure_stderr": 0.0023490297225075376, "rougeLsum_precision": 0.05312130935399373, "rougeLsum_precision_stderr": 0.0019781535292318657, "rougeLsum_recall": 0.4801456290593825, "rougeLsum_recall_stderr": 0.007794216102275114}, "choose the most appropriate solution": {"acc": 0.5048966267682263, "acc_norm": 0.5048966267682263, "acc_norm_stderr": 0.01166526473007815, "acc_stderr": 0.01166526473007815}, "no prompt needed": {"bleu": 0.06614185025884788, "bleu_stderr": 0.006992904774416279, "rouge1_fmeasure": 0.024878431671277717, "rouge1_fmeasure_stderr": 0.0007084095842894381, "rouge1_precision": 0.016184382076826428, "rouge1_precision_stderr": 0.0008939401678502145, "rouge1_recall": 0.14173571994695022, "rouge1_recall_stderr": 0.0030334268104372647, "rouge2_fmeasure": 0.0024039653739287383, "rouge2_fmeasure_stderr": 0.00019957865940879526, "rouge2_precision": 0.0014202808708619317, "rouge2_precision_stderr": 0.00013696486457806, "rouge2_recall": 0.014499650869656043, "rouge2_recall_stderr": 0.0010689452794773893, "rougeL_fmeasure": 0.023587495220817104, "rougeL_fmeasure_stderr": 0.0006681752509678566, "rougeL_precision": 0.015195809153249718, "rougeL_precision_stderr": 0.0008191061120771218, "rougeL_recall": 0.13579609932870865, "rougeL_recall_stderr": 0.0029357166668511343, "rougeLsum_fmeasure": 0.020150738338231722, "rougeLsum_fmeasure_stderr": 0.0005489881127507307, "rougeLsum_precision": 0.012906484148544855, "rougeLsum_precision_stderr": 0.0006299772819920492, "rougeLsum_recall": 0.12046291801697726, "rougeLsum_recall_stderr": 0.002609561210140041}, "pick_correct_choice_index": {"acc": 0.49510337323177367, "acc_norm": 0.49510337323177367, "acc_norm_stderr": 0.01166526473007815, "acc_stderr": 0.01166526473007815}, "what_is_the_correct_ending": {"acc": 0.4956474428726877, "acc_norm": 0.4967355821545158, "acc_norm_stderr": 0.01166557553076037, "acc_stderr": 0.011665382144642394}}, "1": {"Correct the solution": {"bleu": 0.8830608958021205, "bleu_stderr": 0.08562321028264006, "rouge1_fmeasure": 0.040647935659745706, "rouge1_fmeasure_stderr": 0.0014550077414889832, "rouge1_precision": 0.03607709470611491, "rouge1_precision_stderr": 0.0017441028278732172, "rouge1_recall": 0.1952470751009191, "rouge1_recall_stderr": 0.005487103316834906, "rouge2_fmeasure": 0.012117898396108417, "rouge2_fmeasure_stderr": 0.0010508174756117949, "rouge2_precision": 0.007972127932130054, "rouge2_precision_stderr": 0.0008580787510736185, "rouge2_recall": 0.07120120024573734, "rouge2_recall_stderr": 0.004600441634483666, "rougeL_fmeasure": 0.03909307269627734, "rougeL_fmeasure_stderr": 0.0013947783444551163, "rougeL_precision": 0.033979273307145695, "rougeL_precision_stderr": 0.0016175369455391048, "rougeL_recall": 0.19294098495627118, "rougeL_recall_stderr": 0.005472126990576816, "rougeLsum_fmeasure": 0.03550984850410223, "rougeLsum_fmeasure_stderr": 0.001363844016926205, "rougeLsum_precision": 0.03227030109817833, "rougeLsum_precision_stderr": 0.0016612299817095036, "rougeLsum_recall": 0.17402891178983584, "rougeLsum_recall_stderr": 0.0053066479466079685}, "choose the most appropriate solution": {"acc": 0.5016322089227421, "acc_norm": 0.5016322089227421, "acc_norm_stderr": 0.011665762007194868, "acc_stderr": 0.011665762007194868}, "no prompt needed": {"bleu": 0.10718681853330272, "bleu_stderr": 0.018000911234593346, "rouge1_fmeasure": 0.03703813399564072, "rouge1_fmeasure_stderr": 0.0018091336796340791, "rouge1_precision": 0.04132674547577212, "rouge1_precision_stderr": 0.0024485259308753586, "rouge1_recall": 0.08689740514715649, "rouge1_recall_stderr": 0.002916092372055045, "rouge2_fmeasure": 0.005941822792449927, "rouge2_fmeasure_stderr": 0.0006087286220180097, "rouge2_precision": 0.0071242151063781395, "rouge2_precision_stderr": 0.0008842251520965283, "rouge2_recall": 0.011318163060566497, "rouge2_recall_stderr": 0.001006748832876741, "rougeL_fmeasure": 0.03303428861037457, "rougeL_fmeasure_stderr": 0.0015956649904201666, "rougeL_precision": 0.03625394677542022, "rougeL_precision_stderr": 0.0021263148272352293, "rougeL_recall": 0.08136452827721419, "rougeL_recall_stderr": 0.00276242885759181, "rougeLsum_fmeasure": 0.03218887137036506, "rougeLsum_fmeasure_stderr": 0.0016050372004655393, "rougeLsum_precision": 0.03652614662967736, "rougeLsum_precision_stderr": 0.0022174820770640377, "rougeLsum_recall": 0.07577016170577146, "rougeLsum_recall_stderr": 0.0026023305739597473}, "pick_correct_choice_index": {"acc": 0.499455930359086, "acc_norm": 0.499455930359086, "acc_norm_stderr": 0.011665817258899168, "acc_stderr": 0.011665817258899168}, "what_is_the_correct_ending": {"acc": 0.5092491838955386, "acc_norm": 0.5092491838955386, "acc_norm_stderr": 0.011663828032649188, "acc_stderr": 0.011663828032649187}}, "2": {"Correct the solution": {"bleu": 1.4935006686116292, "bleu_stderr": 0.05443694683302151, "rouge1_fmeasure": 0.06032529701781306, "rouge1_fmeasure_stderr": 0.0022263011442644493, "rouge1_precision": 0.05267853270925152, "rouge1_precision_stderr": 0.002449745789546971, "rouge1_recall": 0.24860214672787803, "rouge1_recall_stderr": 0.006500686399418739, "rouge2_fmeasure": 0.023433042203883557, "rouge2_fmeasure_stderr": 0.0018005053734734208, "rouge2_precision": 0.016806908973702062, "rouge2_precision_stderr": 0.0016553589028564524, "rouge2_recall": 0.11666497774928083, "rouge2_recall_stderr": 0.005908602168091971, "rougeL_fmeasure": 0.057693738636468685, "rougeL_fmeasure_stderr": 0.0021627311467575755, "rougeL_precision": 0.04925134882919058, "rougeL_precision_stderr": 0.0023033509345347773, "rougeL_recall": 0.24482036799051923, "rougeL_recall_stderr": 0.006496173955774258, "rougeLsum_fmeasure": 0.05423256001424035, "rougeLsum_fmeasure_stderr": 0.0021531740526494, "rougeLsum_precision": 0.04806464483998574, "rougeLsum_precision_stderr": 0.0023691482012047245, "rougeLsum_recall": 0.2255921129994038, "rougeLsum_recall_stderr": 0.006372394561834973}, "choose the most appropriate solution": {"acc": 0.5048966267682263, "acc_norm": 0.5048966267682263, "acc_norm_stderr": 0.011665264730078145, "acc_stderr": 0.011665264730078145}, "no prompt needed": {"bleu": 0.1712273909909647, "bleu_stderr": 0.029845744864029765, "rouge1_fmeasure": 0.03482910285332883, "rouge1_fmeasure_stderr": 0.0018619215722441118, "rouge1_precision": 0.045080131535418684, "rouge1_precision_stderr": 0.002665199214443293, "rouge1_recall": 0.051600544061536885, "rouge1_recall_stderr": 0.002562420942904496, "rouge2_fmeasure": 0.005293218483445248, "rouge2_fmeasure_stderr": 0.0006663810940536303, "rouge2_precision": 0.006360924839859195, "rouge2_precision_stderr": 0.0008787433792928841, "rouge2_recall": 0.007684278380676342, "rouge2_recall_stderr": 0.000961471039023744, "rougeL_fmeasure": 0.030614463329020847, "rougeL_fmeasure_stderr": 0.0016647185279519317, "rougeL_precision": 0.0394285879549089, "rougeL_precision_stderr": 0.002356887213890825, "rougeL_recall": 0.046948342759142385, "rougeL_recall_stderr": 0.0023931560766638333, "rougeLsum_fmeasure": 0.030839298564677823, "rougeLsum_fmeasure_stderr": 0.0016870567056580634, "rougeLsum_precision": 0.04033700739178538, "rougeLsum_precision_stderr": 0.002458798837565943, "rougeLsum_recall": 0.04587514391567024, "rougeLsum_recall_stderr": 0.002316207868651937}, "pick_correct_choice_index": {"acc": 0.5054406964091404, "acc_norm": 0.5054406964091404, "acc_norm_stderr": 0.011665133500637066, "acc_stderr": 0.011665133500637066}, "what_is_the_correct_ending": {"acc": 0.5108813928182807, "acc_norm": 0.5048966267682263, "acc_norm_stderr": 0.011665264730078145, "acc_stderr": 0.011663061261117737}}, "3": {"Correct the solution": {"bleu": 1.8212244525264603, "bleu_stderr": 0.09849562818662225, "rouge1_fmeasure": 0.0723460691197474, "rouge1_fmeasure_stderr": 0.0026163497556041058, "rouge1_precision": 0.06508938178602254, "rouge1_precision_stderr": 0.002888335161689235, "rouge1_recall": 0.26659804850838303, "rouge1_recall_stderr": 0.006676058649232921, "rouge2_fmeasure": 0.029903412643591754, "rouge2_fmeasure_stderr": 0.0021799636798326538, "rouge2_precision": 0.023241183787932676, "rouge2_precision_stderr": 0.0021132241633176545, "rouge2_recall": 0.13124281702649712, "rouge2_recall_stderr": 0.006161317732310535, "rougeL_fmeasure": 0.06828781879330677, "rougeL_fmeasure_stderr": 0.002518729485584185, "rougeL_precision": 0.05994865949151067, "rougeL_precision_stderr": 0.0026955181247782334, "rougeL_recall": 0.26144434776062075, "rougeL_recall_stderr": 0.006677302291405039, "rougeLsum_fmeasure": 0.06528793058605904, "rougeLsum_fmeasure_stderr": 0.002529000694923454, "rougeLsum_precision": 0.059049668114767744, "rougeLsum_precision_stderr": 0.0027649257923785986, "rougeLsum_recall": 0.24459992364434763, "rougeLsum_recall_stderr": 0.006585924906880039}, "choose the most appropriate solution": {"acc": 0.5272034820457019, "acc_norm": 0.5272034820457019, "acc_norm_stderr": 0.011648545262429021, "acc_stderr": 0.011648545262429021}, "no prompt needed": {"bleu": 0.22910425239933835, "bleu_stderr": 0.023846299745637276, "rouge1_fmeasure": 0.034617382780722655, "rouge1_fmeasure_stderr": 0.001904100064118123, "rouge1_precision": 0.04471001776599436, "rouge1_precision_stderr": 0.0026879796556863317, "rouge1_recall": 0.045685551243527515, "rouge1_recall_stderr": 0.002440523184391023, "rouge2_fmeasure": 0.005307127222312835, "rouge2_fmeasure_stderr": 0.0006509675188338064, "rouge2_precision": 0.006631747554370845, "rouge2_precision_stderr": 0.0009077372437363107, "rouge2_recall": 0.00708411646538401, "rouge2_recall_stderr": 0.0009178652859708077, "rougeL_fmeasure": 0.03053639049565289, "rougeL_fmeasure_stderr": 0.0017070590062506794, "rougeL_precision": 0.03921165897731945, "rougeL_precision_stderr": 0.0023674286187342075, "rougeL_recall": 0.04101390938445623, "rougeL_recall_stderr": 0.0022350756769902065, "rougeLsum_fmeasure": 0.030622384698934416, "rougeLsum_fmeasure_stderr": 0.001718163099004444, "rougeLsum_precision": 0.039886138935695335, "rougeLsum_precision_stderr": 0.002448586484801831, "rougeLsum_recall": 0.04031563564195608, "rougeLsum_recall_stderr": 0.0022007717563761603}, "pick_correct_choice_index": {"acc": 0.515233949945593, "acc_norm": 0.515233949945593, "acc_norm_stderr": 0.011660408257153638, "acc_stderr": 0.011660408257153638}, "what_is_the_correct_ending": {"acc": 0.515233949945593, "acc_norm": 0.5070729053318824, "acc_norm_stderr": 0.011664656918145945, "acc_stderr": 0.011660408257153638}}, "4": {"Correct the solution": {"bleu": 2.0235091580656346, "bleu_stderr": 0.06992480281711495, "rouge1_fmeasure": 0.082090911255068, "rouge1_fmeasure_stderr": 0.002803446342789559, "rouge1_precision": 0.0750764899008003, "rouge1_precision_stderr": 0.003250878714640169, "rouge1_recall": 0.28071950983923893, "rouge1_recall_stderr": 0.00668526364483302, "rouge2_fmeasure": 0.03508516074414074, "rouge2_fmeasure_stderr": 0.0022612345217735254, "rouge2_precision": 0.029562846153838467, "rouge2_precision_stderr": 0.002443964528445172, "rouge2_recall": 0.14150750671815635, "rouge2_recall_stderr": 0.0062328605351622945, "rougeL_fmeasure": 0.07771533962795445, "rougeL_fmeasure_stderr": 0.002693141545803596, "rougeL_precision": 0.06986898167803665, "rougeL_precision_stderr": 0.003066369740945722, "rougeL_recall": 0.2749317176165797, "rougeL_recall_stderr": 0.006684573146158844, "rougeLsum_fmeasure": 0.07481316087741567, "rougeLsum_fmeasure_stderr": 0.0027185549354669615, "rougeLsum_precision": 0.0688802573137848, "rougeLsum_precision_stderr": 0.0031419995471746284, "rougeLsum_recall": 0.2583791181007065, "rougeLsum_recall_stderr": 0.006608286981904223}, "choose the most appropriate solution": {"acc": 0.5125136017410229, "acc_norm": 0.5125136017410229, "acc_norm_stderr": 0.011662170084916898, "acc_stderr": 0.011662170084916898}, "no prompt needed": {"bleu": 0.20107349281952527, "bleu_stderr": 0.04214224677468119, "rouge1_fmeasure": 0.038633264035856554, "rouge1_fmeasure_stderr": 0.0020222776724184797, "rouge1_precision": 0.049366458127839984, "rouge1_precision_stderr": 0.002854014732476892, "rouge1_recall": 0.04923533037170826, "rouge1_recall_stderr": 0.0024982212236816293, "rouge2_fmeasure": 0.005146040515208341, "rouge2_fmeasure_stderr": 0.0006133922293712356, "rouge2_precision": 0.0069147124408860524, "rouge2_precision_stderr": 0.0009836165108404464, "rouge2_recall": 0.0069063816404413315, "rouge2_recall_stderr": 0.0008881448140462662, "rougeL_fmeasure": 0.0341478013156332, "rougeL_fmeasure_stderr": 0.0018057433669060028, "rougeL_precision": 0.043511782572989245, "rougeL_precision_stderr": 0.002558689962342701, "rougeL_recall": 0.04427204057886875, "rougeL_recall_stderr": 0.002279980453794631, "rougeLsum_fmeasure": 0.03455201836586164, "rougeLsum_fmeasure_stderr": 0.0018292045551711296, "rougeLsum_precision": 0.04448265410501512, "rougeLsum_precision_stderr": 0.0026386860903847256, "rougeLsum_recall": 0.04405637966133121, "rougeLsum_recall_stderr": 0.0022677196418328066}, "pick_correct_choice_index": {"acc": 0.514145810663765, "acc_norm": 0.514145810663765, "acc_norm_stderr": 0.011661154475524836, "acc_stderr": 0.011661154475524836}, "what_is_the_correct_ending": {"acc": 0.5114254624591947, "acc_norm": 0.501088139281828, "acc_norm_stderr": 0.011665796539540878, "acc_stderr": 0.011662778026451675}}, "5": {"Correct the solution": {"bleu": 1.9681135945636459, "bleu_stderr": 0.13208724741948602, "rouge1_fmeasure": 0.08562740019962164, "rouge1_fmeasure_stderr": 0.0029691098655082715, "rouge1_precision": 0.08049093429476127, "rouge1_precision_stderr": 0.0035142461324920273, "rouge1_recall": 0.27883210803766567, "rouge1_recall_stderr": 0.006581683130296607, "rouge2_fmeasure": 0.03697008601032397, "rouge2_fmeasure_stderr": 0.002496780466003745, "rouge2_precision": 0.0333369150257513, "rouge2_precision_stderr": 0.0027845366791002944, "rouge2_recall": 0.13700194089143813, "rouge2_recall_stderr": 0.00613423127185248, "rougeL_fmeasure": 0.08076583460906837, "rougeL_fmeasure_stderr": 0.0028795378570412307, "rougeL_precision": 0.07459959278412087, "rougeL_precision_stderr": 0.0033268926264176904, "rougeL_recall": 0.272528207623618, "rougeL_recall_stderr": 0.006590906751190863, "rougeLsum_fmeasure": 0.07760507931200265, "rougeLsum_fmeasure_stderr": 0.002889469047971454, "rougeLsum_precision": 0.0735499511307004, "rougeLsum_precision_stderr": 0.0033973926098363675, "rougeLsum_recall": 0.2546069662211679, "rougeLsum_recall_stderr": 0.006496163637745307}, "choose the most appropriate solution": {"acc": 0.5130576713819369, "acc_norm": 0.5130576713819369, "acc_norm_stderr": 0.01166184537588635, "acc_stderr": 0.01166184537588635}, "no prompt needed": {"bleu": 0.2315660064365379, "bleu_stderr": 0.03985185692641373, "rouge1_fmeasure": 0.03797730190196727, "rouge1_fmeasure_stderr": 0.0019584581823532746, "rouge1_precision": 0.04883644295164218, "rouge1_precision_stderr": 0.0027824290368789364, "rouge1_recall": 0.046722404599889716, "rouge1_recall_stderr": 0.0023869495808978747, "rouge2_fmeasure": 0.005234740517822527, "rouge2_fmeasure_stderr": 0.0006778104688310928, "rouge2_precision": 0.006695738235568391, "rouge2_precision_stderr": 0.0009608876674204776, "rouge2_recall": 0.006591972710456567, "rouge2_recall_stderr": 0.0008806024888411368, "rougeL_fmeasure": 0.0337124518984988, "rougeL_fmeasure_stderr": 0.0017510459198918948, "rougeL_precision": 0.04343865379951131, "rougeL_precision_stderr": 0.002509403663109993, "rougeL_recall": 0.04203034225897826, "rougeL_recall_stderr": 0.0021838302966321445, "rougeLsum_fmeasure": 0.033884080328243846, "rougeLsum_fmeasure_stderr": 0.0017631801582756708, "rougeLsum_precision": 0.04410239398251147, "rougeLsum_precision_stderr": 0.002562032316728619, "rougeLsum_recall": 0.04154031189118946, "rougeLsum_recall_stderr": 0.002165265517853104}, "pick_correct_choice_index": {"acc": 0.5087051142546246, "acc_norm": 0.5087051142546246, "acc_norm_stderr": 0.011664055982032842, "acc_stderr": 0.011664055982032842}, "what_is_the_correct_ending": {"acc": 0.5048966267682263, "acc_norm": 0.4972796517954298, "acc_norm_stderr": 0.011665651503000732, "acc_stderr": 0.011665264730078145}}}, "sciq": {"0": {"Direct Question": {"acc": 0.561, "acc_norm": 0.493, "acc_norm_stderr": 0.015817749561843574, "acc_stderr": 0.015701131345400767}, "Direct Question (Closed Book)": {"acc": 0.296, "acc_norm": 0.312, "acc_norm_stderr": 0.014658474370509005, "acc_stderr": 0.014442734941575018}, "Multiple Choice": {"acc": 0.326, "acc_norm": 0.343, "acc_norm_stderr": 0.015019206922356953, "acc_stderr": 0.01483050720454104}, "Multiple Choice (Closed Book)": {"acc": 0.302, "acc_norm": 0.316, "acc_norm_stderr": 0.014709193056057135, "acc_stderr": 0.01452608023545955}, "Multiple Choice Question First": {"acc": 0.353, "acc_norm": 0.343, "acc_norm_stderr": 0.015019206922356951, "acc_stderr": 0.01512017260548369}}, "1": {"Direct Question": {"acc": 0.578, "acc_norm": 0.543, "acc_norm_stderr": 0.015760691590136388, "acc_stderr": 0.01562562511262066}, "Direct Question (Closed Book)": {"acc": 0.3, "acc_norm": 0.327, "acc_norm_stderr": 0.014842213153411247, "acc_stderr": 0.014498627873361427}, "Multiple Choice": {"acc": 0.329, "acc_norm": 0.34, "acc_norm_stderr": 0.014987482264363937, "acc_stderr": 0.014865395385928357}, "Multiple Choice (Closed Book)": {"acc": 0.32, "acc_norm": 0.343, "acc_norm_stderr": 0.015019206922356953, "acc_stderr": 0.014758652303574874}, "Multiple Choice Question First": {"acc": 0.362, "acc_norm": 0.367, "acc_norm_stderr": 0.015249378464171756, "acc_stderr": 0.015204840912919498}}, "2": {"Direct Question": {"acc": 0.563, "acc_norm": 0.536, "acc_norm_stderr": 0.01577824302490459, "acc_stderr": 0.01569322392873038}, "Direct Question (Closed Book)": {"acc": 0.299, "acc_norm": 0.338, "acc_norm_stderr": 0.014965960710224494, "acc_stderr": 0.014484778521220473}, "Multiple Choice": {"acc": 0.344, "acc_norm": 0.351, "acc_norm_stderr": 0.015100563798316403, "acc_stderr": 0.015029633724408943}, "Multiple Choice (Closed Book)": {"acc": 0.33, "acc_norm": 0.35, "acc_norm_stderr": 0.015090650341444233, "acc_stderr": 0.014876872027456729}, "Multiple Choice Question First": {"acc": 0.374, "acc_norm": 0.38, "acc_norm_stderr": 0.015356947477797572, "acc_stderr": 0.015308767369006358}}, "3": {"Direct Question": {"acc": 0.567, "acc_norm": 0.543, "acc_norm_stderr": 0.015760691590136384, "acc_stderr": 0.015676630912181334}, "Direct Question (Closed Book)": {"acc": 0.301, "acc_norm": 0.321, "acc_norm_stderr": 0.014770821817934637, "acc_stderr": 0.01451239503354315}, "Multiple Choice": {"acc": 0.362, "acc_norm": 0.373, "acc_norm_stderr": 0.015300493622922812, "acc_stderr": 0.015204840912919496}, "Multiple Choice (Closed Book)": {"acc": 0.338, "acc_norm": 0.37, "acc_norm_stderr": 0.015275252316519357, "acc_stderr": 0.014965960710224482}, "Multiple Choice Question First": {"acc": 0.38, "acc_norm": 0.398, "acc_norm_stderr": 0.015486634102858917, "acc_stderr": 0.01535694747779757}}, "4": {"Direct Question": {"acc": 0.573, "acc_norm": 0.557, "acc_norm_stderr": 0.015716169953204105, "acc_stderr": 0.01564978964446221}, "Direct Question (Closed Book)": {"acc": 0.309, "acc_norm": 0.325, "acc_norm_stderr": 0.014818724459095526, "acc_stderr": 0.01461960097720649}, "Multiple Choice": {"acc": 0.356, "acc_norm": 0.36, "acc_norm_stderr": 0.015186527932040127, "acc_stderr": 0.015149042659306623}, "Multiple Choice (Closed Book)": {"acc": 0.345, "acc_norm": 0.346, "acc_norm_stderr": 0.015050266127564446, "acc_stderr": 0.015039986742055235}, "Multiple Choice Question First": {"acc": 0.37, "acc_norm": 0.394, "acc_norm_stderr": 0.015459721957493382, "acc_stderr": 0.015275252316519366}}, "5": {"Direct Question": {"acc": 0.573, "acc_norm": 0.562, "acc_norm_stderr": 0.01569721001969469, "acc_stderr": 0.015649789644462214}, "Direct Question (Closed Book)": {"acc": 0.303, "acc_norm": 0.328, "acc_norm_stderr": 0.014853842487270336, "acc_stderr": 0.014539683710535257}, "Multiple Choice": {"acc": 0.344, "acc_norm": 0.366, "acc_norm_stderr": 0.015240612726405756, "acc_stderr": 0.015029633724408943}, "Multiple Choice (Closed Book)": {"acc": 0.325, "acc_norm": 0.34, "acc_norm_stderr": 0.014987482264363935, "acc_stderr": 0.014818724459095526}, "Multiple Choice Question First": {"acc": 0.375, "acc_norm": 0.374, "acc_norm_stderr": 0.015308767369006363, "acc_stderr": 0.015316971293620996}}}, "story_cloze_2016": {"0": {"Answer Given options": {"acc": 0.5056119722073757, "acc_norm": 0.5125601282736505, "acc_norm_stderr": 0.01155878357073797, "acc_stderr": 0.011561703928784335}, "Choose Story Ending": {"acc": 0.518439337252806, "acc_norm": 0.5456974879743453, "acc_norm_stderr": 0.0115140402455835, "acc_stderr": 0.011554566910658103}, "Novel Correct Ending": {"acc": 0.5008017103153394, "acc_norm": 0.5136290753607696, "acc_norm_stderr": 0.011558135970599896, "acc_stderr": 0.011562417388300206}, "Story Continuation and Options": {"acc": 0.5168359166221272, "acc_norm": 0.5376803848209514, "acc_norm_stderr": 0.011529552555884568, "acc_stderr": 0.011555875693960774}}, "1": {"Answer Given options": {"acc": 0.49812934259754144, "acc_norm": 0.5088188134687333, "acc_norm_stderr": 0.011560633656952961, "acc_stderr": 0.011562351329083266}, "Choose Story Ending": {"acc": 0.5168359166221272, "acc_norm": 0.5339390700160342, "acc_norm_stderr": 0.01153576488164141, "acc_stderr": 0.011555875693960771}, "Novel Correct Ending": {"acc": 0.4863709246392304, "acc_norm": 0.5104222340994121, "acc_norm_stderr": 0.011559920087347773, "acc_stderr": 0.011558135970599896}, "Story Continuation and Options": {"acc": 0.5120256547300909, "acc_norm": 0.5360769641902726, "acc_norm_stderr": 0.01153229486915312, "acc_stderr": 0.011559087533800689}}, "2": {"Answer Given options": {"acc": 0.4949225013361839, "acc_norm": 0.5173703901656868, "acc_norm_stderr": 0.011555452669106635, "acc_stderr": 0.011561836054238776}, "Choose Story Ending": {"acc": 0.51309460181721, "acc_norm": 0.5392838054516301, "acc_norm_stderr": 0.011526690316014594, "acc_stderr": 0.011558466383367176}, "Novel Correct Ending": {"acc": 0.48583645109567075, "acc_norm": 0.5056119722073757, "acc_norm_stderr": 0.011561703928784332, "acc_stderr": 0.01155779233130167}, "Story Continuation and Options": {"acc": 0.5066809192944949, "acc_norm": 0.5264564404061999, "acc_norm_stderr": 0.011546234813777412, "acc_stderr": 0.011561400034509398}}, "3": {"Answer Given options": {"acc": 0.48850881881346875, "acc_norm": 0.5082843399251737, "acc_norm_stderr": 0.01156084507652571, "acc_stderr": 0.011559378273599118}, "Choose Story Ending": {"acc": 0.5056119722073757, "acc_norm": 0.5275253874933191, "acc_norm_stderr": 0.011544898473864576, "acc_stderr": 0.01156170392878433}, "Novel Correct Ending": {"acc": 0.4836985569214324, "acc_norm": 0.49706039551042225, "acc_norm_stderr": 0.01156223242154194, "acc_stderr": 0.011556285484521565}, "Story Continuation and Options": {"acc": 0.4922501336183859, "acc_norm": 0.521111704970604, "acc_norm_stderr": 0.011552120807053817, "acc_stderr": 0.01156104327886354}}, "4": {"Answer Given options": {"acc": 0.4965259219668626, "acc_norm": 0.5163014430785676, "acc_norm_stderr": 0.01155628548452156, "acc_stderr": 0.011562153149168298}, "Choose Story Ending": {"acc": 0.5125601282736505, "acc_norm": 0.5312667022982362, "acc_norm_stderr": 0.011539803085637727, "acc_stderr": 0.011558783570737967}, "Novel Correct Ending": {"acc": 0.4826296098343132, "acc_norm": 0.501336183858899, "acc_norm_stderr": 0.011562390964658753, "acc_stderr": 0.011555452669106632}, "Story Continuation and Options": {"acc": 0.5018706574024586, "acc_norm": 0.5243185462319615, "acc_norm_stderr": 0.011548748301487312, "acc_stderr": 0.011562351329083271}}, "5": {"Answer Given options": {"acc": 0.49331908070550506, "acc_norm": 0.5114911811865313, "acc_norm_stderr": 0.011559378273599126, "acc_stderr": 0.011561400034509398}, "Choose Story Ending": {"acc": 0.5056119722073757, "acc_norm": 0.5275253874933191, "acc_norm_stderr": 0.011544898473864574, "acc_stderr": 0.01156170392878433}, "Novel Correct Ending": {"acc": 0.4890432923570283, "acc_norm": 0.5056119722073757, "acc_norm_stderr": 0.011561703928784332, "acc_stderr": 0.011559655791130734}, "Story Continuation and Options": {"acc": 0.498663816141101, "acc_norm": 0.5232495991448424, "acc_norm_stderr": 0.011549925483927461, "acc_stderr": 0.011562390964658758}}}, "superglue_rte": {"0": {"GPT-3 style": {"acc": 0.5270758122743683, "acc_norm": 0.5018050541516246, "acc_norm_stderr": 0.030096267148976626, "acc_stderr": 0.030052303463143706}, "MNLI crowdsource": {"acc": 0.516245487364621, "acc_norm": 0.5270758122743683, "acc_norm_stderr": 0.030052303463143706, "acc_stderr": 0.030080573208738064}, "does it follow that": {"acc": 0.4368231046931408, "acc_norm": 0.5090252707581228, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.029855247390314952}, "guaranteed true": {"acc": 0.49458483754512633, "acc_norm": 0.5270758122743683, "acc_norm_stderr": 0.030052303463143706, "acc_stderr": 0.030094698123239966}, "should assume": {"acc": 0.5270758122743683, "acc_norm": 0.5270758122743683, "acc_norm_stderr": 0.030052303463143706, "acc_stderr": 0.030052303463143706}}, "1": {"GPT-3 style": {"acc": 0.49458483754512633, "acc_norm": 0.49097472924187724, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.030094698123239966}, "MNLI crowdsource": {"acc": 0.49097472924187724, "acc_norm": 0.49097472924187724, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.030091559826331334}, "does it follow that": {"acc": 0.49097472924187724, "acc_norm": 0.49097472924187724, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.030091559826331334}, "guaranteed true": {"acc": 0.49097472924187724, "acc_norm": 0.48375451263537905, "acc_norm_stderr": 0.030080573208738064, "acc_stderr": 0.030091559826331334}, "should assume": {"acc": 0.49097472924187724, "acc_norm": 0.49097472924187724, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.030091559826331334}}, "2": {"GPT-3 style": {"acc": 0.49458483754512633, "acc_norm": 0.5018050541516246, "acc_norm_stderr": 0.030096267148976633, "acc_stderr": 0.030094698123239966}, "MNLI crowdsource": {"acc": 0.4981949458483754, "acc_norm": 0.48736462093862815, "acc_norm_stderr": 0.030086851767188564, "acc_stderr": 0.030096267148976633}, "does it follow that": {"acc": 0.48736462093862815, "acc_norm": 0.48375451263537905, "acc_norm_stderr": 0.030080573208738064, "acc_stderr": 0.030086851767188564}, "guaranteed true": {"acc": 0.47653429602888087, "acc_norm": 0.47653429602888087, "acc_norm_stderr": 0.03006330041190266, "acc_stderr": 0.03006330041190266}, "should assume": {"acc": 0.48014440433212996, "acc_norm": 0.48014440433212996, "acc_norm_stderr": 0.0300727231673172, "acc_stderr": 0.0300727231673172}}, "3": {"GPT-3 style": {"acc": 0.49458483754512633, "acc_norm": 0.49458483754512633, "acc_norm_stderr": 0.030094698123239966, "acc_stderr": 0.030094698123239966}, "MNLI crowdsource": {"acc": 0.4981949458483754, "acc_norm": 0.48375451263537905, "acc_norm_stderr": 0.030080573208738064, "acc_stderr": 0.030096267148976633}, "does it follow that": {"acc": 0.47653429602888087, "acc_norm": 0.4548736462093863, "acc_norm_stderr": 0.029973636495415252, "acc_stderr": 0.03006330041190266}, "guaranteed true": {"acc": 0.49458483754512633, "acc_norm": 0.4657039711191336, "acc_norm_stderr": 0.030025579819366426, "acc_stderr": 0.030094698123239966}, "should assume": {"acc": 0.48375451263537905, "acc_norm": 0.49458483754512633, "acc_norm_stderr": 0.030094698123239966, "acc_stderr": 0.030080573208738064}}, "4": {"GPT-3 style": {"acc": 0.48014440433212996, "acc_norm": 0.4548736462093863, "acc_norm_stderr": 0.029973636495415252, "acc_stderr": 0.0300727231673172}, "MNLI crowdsource": {"acc": 0.5054151624548736, "acc_norm": 0.5054151624548736, "acc_norm_stderr": 0.030094698123239966, "acc_stderr": 0.030094698123239966}, "does it follow that": {"acc": 0.4729241877256318, "acc_norm": 0.49458483754512633, "acc_norm_stderr": 0.030094698123239966, "acc_stderr": 0.030052303463143706}, "guaranteed true": {"acc": 0.49097472924187724, "acc_norm": 0.49458483754512633, "acc_norm_stderr": 0.030094698123239966, "acc_stderr": 0.030091559826331327}, "should assume": {"acc": 0.48014440433212996, "acc_norm": 0.5018050541516246, "acc_norm_stderr": 0.030096267148976626, "acc_stderr": 0.030072723167317194}}, "5": {"GPT-3 style": {"acc": 0.48736462093862815, "acc_norm": 0.47653429602888087, "acc_norm_stderr": 0.03006330041190266, "acc_stderr": 0.030086851767188564}, "MNLI crowdsource": {"acc": 0.48375451263537905, "acc_norm": 0.5018050541516246, "acc_norm_stderr": 0.030096267148976626, "acc_stderr": 0.030080573208738064}, "does it follow that": {"acc": 0.4620938628158845, "acc_norm": 0.4693140794223827, "acc_norm_stderr": 0.030039730592197812, "acc_stderr": 0.030009848912529117}, "guaranteed true": {"acc": 0.4981949458483754, "acc_norm": 0.4981949458483754, "acc_norm_stderr": 0.030096267148976626, "acc_stderr": 0.030096267148976633}, "should assume": {"acc": 0.4981949458483754, "acc_norm": 0.5090252707581228, "acc_norm_stderr": 0.030091559826331334, "acc_stderr": 0.030096267148976633}}}, "winogrande": {"0": {"Replace": {"acc": 0.5059194948697711, "acc_norm": 0.4940805051302289, "acc_norm_stderr": 0.014051500838485807, "acc_stderr": 0.01405150083848581}, "True or False": {"acc": 0.4980268350434096, "acc_norm": 0.5082872928176796, "acc_norm_stderr": 0.014050555322824192, "acc_stderr": 0.014052376259225636}, "does underscore refer to": {"acc": 0.4877663772691397, "acc_norm": 0.4846093133385951, "acc_norm_stderr": 0.014045826789783665, "acc_stderr": 0.01404827882040562}, "stand for": {"acc": 0.4925019731649566, "acc_norm": 0.48855564325177586, "acc_norm_stderr": 0.014048804199859325, "acc_stderr": 0.01405090552122858}, "underscore refer to": {"acc": 0.48855564325177586, "acc_norm": 0.4925019731649566, "acc_norm_stderr": 0.01405090552122858, "acc_stderr": 0.014048804199859329}}, "1": {"Replace": {"acc": 0.49329123914759276, "acc_norm": 0.500394632991318, "acc_norm_stderr": 0.014052481306049516, "acc_stderr": 0.014051220692330349}, "True or False": {"acc": 0.4940805051302289, "acc_norm": 0.494869771112865, "acc_norm_stderr": 0.014051745961790516, "acc_stderr": 0.01405150083848581}, "does underscore refer to": {"acc": 0.4877663772691397, "acc_norm": 0.49013417521704816, "acc_norm_stderr": 0.014049749833367589, "acc_stderr": 0.01404827882040562}, "stand for": {"acc": 0.500394632991318, "acc_norm": 0.4980268350434096, "acc_norm_stderr": 0.01405237625922564, "acc_stderr": 0.014052481306049516}, "underscore refer to": {"acc": 0.48539857932123126, "acc_norm": 0.48697711128650356, "acc_norm_stderr": 0.014047718393997663, "acc_stderr": 0.014046492383275835}}, "2": {"Replace": {"acc": 0.4877663772691397, "acc_norm": 0.5090765588003157, "acc_norm_stderr": 0.014050170094497704, "acc_stderr": 0.014048278820405621}, "True or False": {"acc": 0.5035516969218626, "acc_norm": 0.500394632991318, "acc_norm_stderr": 0.014052481306049516, "acc_stderr": 0.014052131146915845}, "does underscore refer to": {"acc": 0.5201262825572218, "acc_norm": 0.4988161010260458, "acc_norm_stderr": 0.014052446290529019, "acc_stderr": 0.014041096664344327}, "stand for": {"acc": 0.5169692186266772, "acc_norm": 0.5027624309392266, "acc_norm_stderr": 0.014052271211616445, "acc_stderr": 0.01404439040161298}, "underscore refer to": {"acc": 0.4925019731649566, "acc_norm": 0.500394632991318, "acc_norm_stderr": 0.014052481306049516, "acc_stderr": 0.014050905521228573}}, "3": {"Replace": {"acc": 0.505130228887135, "acc_norm": 0.5074980268350434, "acc_norm_stderr": 0.014050905521228571, "acc_stderr": 0.014051745961790516}, "True or False": {"acc": 0.5059194948697711, "acc_norm": 0.5019731649565904, "acc_norm_stderr": 0.014052376259225629, "acc_stderr": 0.014051500838485807}, "does underscore refer to": {"acc": 0.5122336227308603, "acc_norm": 0.5035516969218626, "acc_norm_stderr": 0.014052131146915857, "acc_stderr": 0.01404827882040562}, "stand for": {"acc": 0.5122336227308603, "acc_norm": 0.5035516969218626, "acc_norm_stderr": 0.014052131146915857, "acc_stderr": 0.014048278820405616}, "underscore refer to": {"acc": 0.5074980268350434, "acc_norm": 0.4956590370955012, "acc_norm_stderr": 0.0140519560640769, "acc_stderr": 0.01405090552122858}}, "4": {"Replace": {"acc": 0.510655090765588, "acc_norm": 0.5035516969218626, "acc_norm_stderr": 0.014052131146915852, "acc_stderr": 0.0140492945362904}, "True or False": {"acc": 0.526440410418311, "acc_norm": 0.5232833464877664, "acc_norm_stderr": 0.014037241309573645, "acc_stderr": 0.014032823874407224}, "does underscore refer to": {"acc": 0.5114443567482242, "acc_norm": 0.4980268350434096, "acc_norm_stderr": 0.014052376259225629, "acc_stderr": 0.014048804199859322}, "stand for": {"acc": 0.5098658247829518, "acc_norm": 0.489344909234412, "acc_norm_stderr": 0.014049294536290403, "acc_stderr": 0.014049749833367596}, "underscore refer to": {"acc": 0.500394632991318, "acc_norm": 0.49171270718232046, "acc_norm_stderr": 0.014050555322824192, "acc_stderr": 0.014052481306049516}}, "5": {"Replace": {"acc": 0.505130228887135, "acc_norm": 0.4964483030781373, "acc_norm_stderr": 0.014052131146915867, "acc_stderr": 0.014051745961790516}, "True or False": {"acc": 0.5122336227308603, "acc_norm": 0.5090765588003157, "acc_norm_stderr": 0.0140501700944977, "acc_stderr": 0.01404827882040562}, "does underscore refer to": {"acc": 0.489344909234412, "acc_norm": 0.49171270718232046, "acc_norm_stderr": 0.014050555322824192, "acc_stderr": 0.014049294536290393}, "stand for": {"acc": 0.5090765588003157, "acc_norm": 0.48303078137332284, "acc_norm_stderr": 0.014044390401612972, "acc_stderr": 0.014050170094497712}, "underscore refer to": {"acc": 0.4988161010260458, "acc_norm": 0.49013417521704816, "acc_norm_stderr": 0.014049749833367596, "acc_stderr": 0.014052446290529019}}}}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..1df34fdfdececf50e44d26d031f61117cf09874d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.1295929648118789,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.017717451610532654
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.057688485079104755,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002659417350231105
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.18771361409144396,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.004194847117781067
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07130151635044231,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017145172534413066
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.022591789424650413,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0012409453957858067
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.08711168383171403,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0026849545852757613
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.031734228703201264,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010224014054478407
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05685991375829496,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0026473311931234454
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.18500805358362152,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.004147431246244616
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.0701415239947909,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016880620041233863
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.05595104569982353,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002636691350690872
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1806575902205997,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003940723272696618
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.068682573498121,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001637051783542573
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..de7b0b7bfac147cb65e4b2eab9eb953821b7735a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.14987013983247244,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.01941974811684623
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.05507727706640666,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019564677497519853
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.18027308616485552,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0038002016718867975
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07380028517409874,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017644455999507468
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.020536550124082592,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009198527068768572
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07773785714938754,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0024576072009639223
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.029924776302610284,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010166921536067964
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05266527434294661,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0018768852497859027
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.1755608787259102,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0037271693076169517
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07089555573728011,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016677661366070384
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.05264662810449602,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001884149706620751
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.17473281354464437,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0036722904268852133
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07071172213779651,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001659838827248763
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..74fc304748e471a8766d2b26f74e725cdf1c5ceb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.12302264952010052,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.008473608033646678
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.055679194052293414,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015790115772431718
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.1808141666654767,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003768554209591496
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.0768642756778079,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001811200981366701
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.02070146702208313,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007492369519392376
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07660233137719231,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00242307758789654
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.030198905160389044,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.000982928967923372
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.052039959674758364,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001392609794321448
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17496136681829913,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.003685600912041933
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07286820570794587,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016717956031784844
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.052645110361589434,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001455115958473737
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.17488088925366627,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003655243712914895
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07325502354058079,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001694053285569078
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c5de75117216e3a17d4fb70830bb028fefa09a8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.1494775278828044,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.022242668837387642
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.05819590767349479,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.00187266941961563
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.1837211769068863,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037631739340242853
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07759644830725707,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017814677119907972
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.020971065102809404,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.000895456781665476
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.078084178799313,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0024146049927506823
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.0299181905955771,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009706547160371815
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05435903919389146,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001720611632601612
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17679767348251002,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0036809106852510385
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07316137688753947,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016471496270948857
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.05492316378292939,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001744093415859728
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.17690363183304764,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003638138098577823
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07366095409257907,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016586575850230274
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a694093b1b304251eac6fee438c220d749d19f25
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.19164606319365804,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.024842965498983755
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.05993715403649896,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0016748088373488277
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.18999825922564922,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037155688461071694
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.08218073938962703,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018499322659086194
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.021877668010459907,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007551954975307327
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07987598814280476,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0023972171750080043
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.031639571635299846,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009862002820496016
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05569962429765539,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014542809726584018
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.1829897393194941,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.003626975042403296
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07749057699272385,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016921799932615024
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.056374413163303896,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015045460779383418
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1831731672325477,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003597143627516616
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07799095859947036,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017129521717618506
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c4f7442b18eec615b5d5eae73d765fa47089ed15
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_PALM_prompt_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.17574933687846614,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.025384181870486397
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.05972888483104343,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0018329849074038712
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.18292682744906683,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003663997330795685
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07946226594400384,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018412019107524114
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.021074770698868366,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007683455677750384
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07513861469355282,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.002333692457604091
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.030022553056029706,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009742418086937908
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05516127047546632,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0015392382782714019
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17614313937858433,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0035646877950734107
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07483720907439476,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001672817823692251
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.055368021748712734,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001561582516111846
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1756797982007548,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003531535198778665
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07489823136944627,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016865681248032596
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5a78f1d63dbe2b42b3cf8bacd41ce049f537602
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.02716794991360227,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0008204423792394508
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.1822527382056869,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002246189358522882
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.04370443194320921,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0008270949225797383
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.001333594787639512,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00011638013641618982
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.009285831703201557,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0005095411845886788
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.0021022925642461623,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00011341026500721504
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.026796470589552018,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0007542828958134997
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.1812832654319008,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002242575337885422
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.043375077921510685,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008125955785869596
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.019859301830203156,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.000705291344078074
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.13504875184894552,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0017157651520892864
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.031579687187122944,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.000632968954798297
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.007162912027319248,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 8.799732134630633e-05
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6df30397b3658ca7efa4c0ad6a4395cdb40db1f4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.16883673090259363,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002502030010371229
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.15618300732830487,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002359067976848311
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.14164576016651942,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001778754222107073
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.01672633503624124,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0011081302796613816
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.01600514703432178,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0009715621757517813
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.013624757551384124,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007818220844644996
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.14224381757445975,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.00213034490021511
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.13192814705914896,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00200227535288082
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.11830599709144449,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014079740228845011
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.15042428668427457,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002260006746662012
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.13825539582157836,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002041174554406684
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.1252632960833956,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015337180547933837
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.6590131747530226,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.12977618045440018
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ab30a52e009ad75bc3c1b893eae21ef2622530ca
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.1705969694142156,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0025131229098136585
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.15761321437356865,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002432157802246827
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.14354986777463272,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018339278799392887
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.017122190675118196,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0012447818915600876
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.016763144168686534,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001041171249249909
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.014100180203474038,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008276615210277857
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.14270089197334904,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002135234458917749
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.1326975552677369,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0020774353171695834
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.11949064473731222,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001470601686701589
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.1515024612128631,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002270510371981427
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.1392146312336204,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002115600656748789
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.12656168844098892,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015798154653780215
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.5492689650034025,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08789814958865551
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..2120d83522eccf7cab9e6e024039d83517a62c95
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.16990770521290732,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0025438057474100073
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.16021997486913941,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0024160108734416854
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.1441015790918314,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018602827624463706
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.015214536900019273,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010692205053183784
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.015420185258765253,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0010273736824902339
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.013249922136095603,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008649946361473154
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.14196899698534282,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021319224801637135
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.13510011259495838,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0020902509105043673
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.11986132557556263,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001509826051165344
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.15080440042366344,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022898520672142444
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.14098767187068265,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021167327935460206
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.1268605469148488,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016208332554181627
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.5225086157392205,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08782464964651336
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7dfcdf3336ce5b5ed192362f0f08d1a7e240085
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.17001348002992608,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002472945726795613
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.16641335439404173,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002516334305490709
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.14606482836681436,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018280556446259094
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.01591640786398681,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.000961538418662567
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.017920512300572296,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001111692174103912
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.014229702953664455,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008237983700670122
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.14223869415288778,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020766661579404897
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.14080246405083494,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0021890583449121997
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.12175024842032746,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014788274431685848
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.1506981914188859,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.00224238363239455
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.14600473949937198,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021735849443758988
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.12836641711166125,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015963226153607253
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.5342732499256956,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.1027315904453386
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..646f04cba6ee47c0208b767a76fc80bfadfe6fc1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_explicit-graph-description2_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_precision": 0.17335244138244874,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002628810045816908
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_recall": 0.16573763442748227,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002438138421190895
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge1_fmeasure": 0.14725933348138234,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018907376621132115
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_precision": 0.018051903585621448,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0011163727106672703
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_recall": 0.017828395181452815,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0011751907098589418
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rouge2_fmeasure": 0.015163133405164434,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009412775453260759
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_precision": 0.1443998895689471,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021924349422918392
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_recall": 0.13967945448595898,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002066115416965032
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeL_fmeasure": 0.12234405519301784,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.00149721265164285
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_precision": 0.15358706687881649,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023486595525211425
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_recall": 0.14542801653317822,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0020859852388272707
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "rougeLsum_fmeasure": 0.12944852616669666,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001605797602788086
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "explicit-graph-description2",
+      "bleu": 0.5599756287973008,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08624872080359487
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..16ba68ad0fbfce6fcca31910044f274fbf845254
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.04787169131286324,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.01817594262932824
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.05219573618812013,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0036463591354812734
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.2092941055787021,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002738958078725276
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.04686425613621658,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0008219392538504159
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.0023893079304512085,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00017566116680500324
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.020431268269765074,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0011345589198703068
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.004097730822753029,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.000284378780561254
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.05149194714448206,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0036328394668563605
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.2070204298599423,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0026942905482556577
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.04594073467891509,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0007486823064547486
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.044498915692550794,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0036541031800773204
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.15356732693991013,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021296241897594786
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.03389173137151838,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0006685380625775191
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3350420fb5fb9eed00ef85e745c2981a3b232eca
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.6245795597079693,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.12364633491228348
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.16788566040200872,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002506066712601704
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.156107822389262,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023580502662707355
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.1408351138429239,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017939649759145585
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.016831623170465607,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.001109690767212444
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.016312218393737095,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0009932711354024328
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.013747268988959595,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007853529846111664
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.14146179896174868,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.00213302407544127
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.1319410650215301,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.001993005317327167
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.11764346217525097,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014184921221509462
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.14956397844019953,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022662888334540306
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.13813269733237477,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002042887172507199
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.12449737336986157,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015465935152696479
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..32be3ccaa87dd33a12645bbd11015801ea0d0795
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.44030050792600856,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08918965531864674
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.165020909705563,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002497799117783653
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.1579071204222324,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0024816328723868283
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.14063207331011088,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018732758899568784
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.015020754513726486,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009072131098635255
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.016692167939649796,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0010286596188788658
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.013404518942651407,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007787079973307792
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.13749360940884617,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020803253174644953
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.13328479413864802,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002139290960697942
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.11682645738111967,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014963127749698607
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.1464246392027477,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022481093268697607
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.1390216810249052,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021491921792093326
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.1239705878552023,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016213406765116238
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea8e9186d87f95072c0a5c0c67671bc724b09a99
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.4028004322246814,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.09627992264831524
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.16192215223479117,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0025886791288695856
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.1591737175072444,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002676280527776092
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.13807126726930546,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001961867694651278
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.01532497701640921,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00096053634684961
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.01789090329021704,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001138291110706073
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.013694547280363368,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008406341572135305
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.1348196406800519,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021490241206731956
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.13491070126011628,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0023624548127525523
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.11455050831050843,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001572738062460073
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.1434457740106499,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023224644970998463
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.13929199647413618,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.00233665463674003
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.12128174476152666,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017094370940622802
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0c2399a792444cfd639df5f411de930ae59dff4f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.48038870322086485,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.0950744978183185
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.1612689936412253,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002593706132714892
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.16230890102132264,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0027329862790525937
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.13898420590680097,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002005636475842531
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.01639943402464418,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010462999278646942
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.019047005353722873,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0012346130129613677
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.01467598496147871,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.000943689965606244
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.13528835328741515,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021828816198137323
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.1384797370898447,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0024281776883333106
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.11632551379631219,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016469373399055434
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.1433977750531889,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023475946507656165
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.14234860995071957,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0023567460311511336
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.12238496569136655,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017501406428895951
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e68fb8b15d6ba8be30805e62f2be9fe564f19a3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_implicit-graph-description_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "bleu": 0.45115235727169706,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.07183544899400104
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_precision": 0.16524773113217792,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002761577061548867
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_recall": 0.1671900838419029,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0028680251377737
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge1_fmeasure": 0.14037246602351963,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002008576466387765
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_precision": 0.01852378604139675,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.001108714616606197
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_recall": 0.02208117704754726,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014164098810313566
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rouge2_fmeasure": 0.01573629832661093,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009296010590017547
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_precision": 0.13842446049242282,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0023215511141633505
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_recall": 0.14325804112156512,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002575571482543266
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeL_fmeasure": 0.1174214317896419,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016188198074821145
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_precision": 0.14624932204164548,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0024721564063546987
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_recall": 0.14630209531166793,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002457955701083795
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "implicit-graph-description",
+      "rougeLsum_fmeasure": 0.12334767454389015,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001735345048925764
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0099371a8f829cc7b11a6ec4eab7be5ab0d248fd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.024257094385346158,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0012542520013724245
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.14438146327799528,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002210767634374098
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.03521658777407273,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001032855438532496
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.0036895014079609267,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0006419014802940976
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.01251292918426236,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0012328752315072008
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.004339378512582052,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006544799388018146
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.022267351364883243,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001060310841195819
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.137175853341158,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0021144498099671533
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.03281178600545513,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008979493883162579
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.021691614849279027,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0011300013638413863
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.1294165016082978,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0019946881884391385
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.03131853952441433,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.000904933217930217
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.011683013048448883,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.002299898655396354
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa7ec5a005a44945d48ecf86f69b12ed97c94c9b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.166421902281267,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002509536624513244
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.15640224145913584,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023886985315635617
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.14008242251401395,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017910410554211656
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.016574599293167843,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.001097074412480475
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.016727604524752332,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0010405236023368633
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.013642298532841417,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00077960580664087
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.14022832153745743,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.00213486067462865
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.13246983853698344,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002033559600741525
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.11709634674078381,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014146593664078942
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.14818197139020592,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002271024873481679
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.13825385118721117,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002074894761466916
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.12378978270420449,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015474580156464902
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.6162564327220701,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.1172533643555817
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d958143e5911c715198a9ed3cda6669b2fda3a25
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.16319819867642651,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002426464101736119
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.15404042754151479,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002478306564411953
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.13892759709779767,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018265728869274942
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.014912982683458914,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009237093791681929
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.016297943525760927,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0010649358289921388
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.013089719060620607,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007726454211608279
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.13741035910079322,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020428354842843533
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.13057150223971264,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0021219065821278115
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.11650832190139555,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014759630408007285
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.14553524668113407,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022002606329358026
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.13620255058111988,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021639599169587793
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.1229747664140421,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015963298971135365
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.464666510911833,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08222220573011234
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..a36d1ed5050300f006392581af346c62825c515d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.16188795072384957,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0025606178815772186
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.14779323773320033,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0025010089208298046
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.1351641368693967,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0019474873765221823
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.015524787502426208,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.001042198532546253
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.015185223617908139,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00114853509785443
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.013153025539986579,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008953407702700355
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.13524306101257844,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002145203133780158
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.12387529380771088,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002083956552126564
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.11226025544826589,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015601459007713312
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.14350630370022732,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022925759452389416
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.12941050420184938,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002157229969105682
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.11871298212567302,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016806710065392066
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.5347923980216038,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.05654660025438367
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..2624102666bf9c90816ab6aef375fe49e820753f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.1643685889453247,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0024901757152926327
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.15465227513600494,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0025856665354129014
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.13890191242592734,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018984947282986685
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.015811707006913117,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008876824189595942
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.017023747384715765,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001064465606217159
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.013872966659414053,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007622040129882122
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.13761313588982987,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002106009553028678
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.1298443924561633,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002193510884352127
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.11546828539245778,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015269267163462042
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.1458255331490558,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022720293179680218
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.13542456041940654,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0022271994295923335
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.12188337669823847,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016518640850592327
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.447830278815385,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.06715101669775739
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a521be1e297a639f47e32e9e6a1c4a13817cc849
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_non-explicit-description_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_precision": 0.1646198127817207,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002646484323779933
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_recall": 0.1490304755831364,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023689773981536632
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge1_fmeasure": 0.13625801392457523,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.00187042901923777
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_precision": 0.01713122779441091,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010058504976427649
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_recall": 0.015592738904389145,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0009494920127045856
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rouge2_fmeasure": 0.013595010492421218,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007463780707733565
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_precision": 0.13886708223114388,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002259848022664383
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_recall": 0.12602515814645163,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019898391778165677
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeL_fmeasure": 0.11419438860339491,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015164933018915136
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_precision": 0.14703846057442585,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0024248196702567554
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_recall": 0.13191735359035292,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0020719729544053833
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "rougeLsum_fmeasure": 0.12064027783181062,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001640982408038248
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "non-explicit-description",
+      "bleu": 0.4758464184876175,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.10146555413589751
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa1de441dfdb1f1079caa8255c4223f6c92d4ffc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.08308142158928723,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0014822533733938113
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.4813530094293165,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0052207165316912555
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.1338747913042839,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002030240874957092
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.029535536265787973,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008372816447783799
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.17908532550201076,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.003415407692561151
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.04755242841283524,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0011842958136442528
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.07204203263806305,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011304033839566587
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.44548440426035085,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00514873304330449
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.1175214238809207,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015866825442232382
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.06951673475733636,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0013120942001119277
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.40195435934423007,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.004355224640972606
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.11176670755192196,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001789832056695336
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.4105317095314889,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.043066710267683765
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..aeb71d2eeeab4d3d0927e993c608d7c4ee1916cd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.1662782777493377,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0025085144836969712
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.16758435156210738,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0030206942122441363
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.14107370791796536,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017982597280216904
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.017259872587569457,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0011062856087889594
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.022404809879098816,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001426195880760084
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.014903773827883306,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008144412261826056
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.1398613211175107,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002132190748276612
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.14190066206999388,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0026252171398360932
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.11763972203198278,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014171066452616822
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.14788595860466908,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022677650604797407
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.1476217188383832,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002589681176386657
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.12442129008035598,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015468255706592889
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.6314175464516708,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.12498388493367703
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dc44d200efbc065c0e6d9a08cb5f71c814f9eee
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.1653712229197853,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002440924984163296
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.1653250915896547,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0029135773698179563
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.14125838813948505,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018147591314905
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.01650829521165785,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010046369743166356
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.020600281994805424,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013239499542895858
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.014494882572376312,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008275950679432722
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.13917888276805995,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020659023219378315
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.14043061866293993,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002517638410671949
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.1184115219641423,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.00146376498152489
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.1471096818527704,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022017203919913464
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.14601316919655402,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0025452049628687883
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.1247729325662712,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001572783517385721
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.5931779549207392,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.09339568633223094
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..c8d52017edcc65f8940d39786b830290c8914e02
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.16582665337512495,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0024208349959723994
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.16255706599105907,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002834223972206501
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.13960615389268183,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017523193011045669
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.015177432476958089,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008927376595589448
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.019082870994391096,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001300150390527605
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.013187050625056523,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007211908914592551
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.13831816790209567,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020310042200119573
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.13727289920306487,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0025055915595956126
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.11581175106585567,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0013986409084470917
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.14670366020679268,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.00216023102430518
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.1428602401569226,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002485596883393547
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.12243760940103116,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0014992182881352384
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.613415825073411,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.15363249845233518
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..7ae8ecab0e418382ce5b229ac86928b714c21bfc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.16440247158022403,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002408134958644391
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.1672878148244216,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0028544677726046685
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.1415749443824625,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018144061370138271
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.016582657589509987,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008859080182706377
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.02110856663946593,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001326962436284589
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.014802883130151939,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007502428586822482
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.13771889673883428,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002017980024471294
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.14130600028610976,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0024909401470214177
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.11783817259815357,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001441797703688301
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.14564352836992275,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002182392381771102
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.14647550822693875,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002455629059429704
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.12419545819826318,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015690813210837236
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.5374477526028321,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.09283692815634105
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..685191212be236c5aea7342660e3e4125d636892
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-web_nlg_en_very-explicit-description_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_precision": 0.16597024457966064,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0026243195088935236
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_recall": 0.16986441790095472,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0030299534520702256
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge1_fmeasure": 0.14047911280614023,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001918909621032034
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_precision": 0.018689456645170934,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.001121139158372213
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_recall": 0.023799423953292702,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015503997097516637
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rouge2_fmeasure": 0.01624180543052662,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009504654937149193
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_precision": 0.13981148514085578,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002224344509680242
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_recall": 0.14559045715327285,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0026961659834251164
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeL_fmeasure": 0.11793359101554969,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015470077154010478
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_precision": 0.14803607615354483,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023878068281125613
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_recall": 0.14963478549748452,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0026025633576893647
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "rougeLsum_fmeasure": 0.12410304209348472,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016642202118066153
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "very-explicit-description",
+      "bleu": 0.5504965404148479,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.080427930138377
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f8ef4d2b4b06257aecf370103ff46849eebb7673
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.1890231240274845,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019684320777096703
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.339017718024998,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0028323433133959483
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.22563848251054694,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018751473618148513
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.04380838750833082,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008419429245577659
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.08179849168709151,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0017042047474535047
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.052567975188848405,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009556547743829917
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.127676216716478,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012153296324758198
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.23953125612828238,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0022402139432793856
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.15451828388176664,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001208772883015503
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.17481696246700595,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0018160527235596866
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.31443864114060294,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0026582156582863913
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.20883538497621398,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017334509196587264
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 2.3244260259071865,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.07286281897100536
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b4e0db5e4c7572b049c88bead742a44320e5cc12
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.16030602957044252,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001967524581211362
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.272400210100343,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003051643168753336
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.18474230303970723,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0019632536598955266
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.03166192462922729,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007933357990075433
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.05886551805358644,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015884469531505948
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.03761257717991928,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0008913626176034577
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.11291539782776717,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012748009948283148
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.19815104405287912,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002316563028814676
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.13089239358773688,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0012729550476269687
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.14915402681793893,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0018104204962086678
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.25378286760463403,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0028305419282366193
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.1719442252457088,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001807476862548232
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 1.828441133091315,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.04797244163227974
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c85394816aaaa00b3254ff0c21560db9ec522416
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.13691057056766903,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001825265602905107
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.22539218166685782,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002822899330851212
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.1553013724184807,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001829739118077522
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.02181756499211645,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007137545292173706
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.039804717049478786,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013560991274283968
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.025570973012891195,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007692752377271742
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.10068399069926871,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012243521909029603
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.17123882520643308,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002168643066220908
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.11505946082280019,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0012278934893675208
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.12780542015362023,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016861150014377962
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.21034780895799102,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002605948309335639
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.14484893653987274,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016765021056686803
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 1.319957729947558,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.055949786634926266
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb85480d12fcb3f00a75272dc0de82d640d0e6ae
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.11331660679440218,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0020148980956489313
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.17682573132355542,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0029418843134879813
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.12309846747867732,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0019272106431180854
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.018033312403096684,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0006783067936208244
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.031206199578028934,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001235176208046927
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.02030504236701163,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0007026247733576691
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.0867545256195891,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0015118580414631715
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.13772453378421665,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.002277790929230465
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.09410057985901603,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0013756456156665128
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.10596910703751164,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001891907064541939
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.16492851491383875,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0027267448285863516
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.1147938850890483,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017791199568731235
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 1.2271649835407419,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.061326948665848736
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..5dd251c18f85edb47ed85f742d49cf0995cb3185
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.038476995951992485,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0016453155252656947
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.057681790969237874,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023058359041284843
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.039273927677435934,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0015037043115189227
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.006055290873763252,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0004556065968758473
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.011582702540716703,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0009372131400483686
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.006731993501111477,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00045966773056801204
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.0299699971593133,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012852804923565745
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.04584513362616462,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018409273324060942
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.030451500169627634,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0011231733238070457
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.03588852826032929,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015400770868070532
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.05364153027915661,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002137335190836827
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.036456484857113516,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013830180281845924
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 0.21901783251929063,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.023129015503409796
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..e2e66522f6f389a4504fdd03364d90426cab4e58
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_article_summary_en_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_precision": 0.00741304726602641,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0009755620107809973
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_recall": 0.009480629169855271,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0010572884760238043
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge1_fmeasure": 0.006323793453713114,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.000679766707607541
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_precision": 0.001214496073168751,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00022011821368556766
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_recall": 0.0021696621911090567,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00045950435492140597
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rouge2_fmeasure": 0.0012665122562968891,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00021596794467997575
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_precision": 0.006028681674848774,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0008454472955138656
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_recall": 0.007771898833374424,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0008802017587782391
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeL_fmeasure": 0.004971201029477744,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0005078524339863577
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_precision": 0.007036952939226733,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0009413610021170078
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_recall": 0.008965157586393747,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0009950047096009401
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "rougeLsum_fmeasure": 0.005929594045281956,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0006291046983741132
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "article_summary_en",
+      "bleu": 5.322880476060286e-08,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 1.5191889091292856e-07
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f8e5acc795f003628bc74ad8426af698ff17d8f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.07162315852179309,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0014555411662114063
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.1024819369868203,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0019137149698756734
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.07728270900364194,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001418281974629439
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.007210468394688485,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00037775068659526225
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.010657999963685463,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.000602849822298443
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.007854289251701565,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00040869938896478014
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.06359760111567767,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012025425436729914
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.09274514173812211,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0016759276752536403
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.06907689906047816,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0011853028745420664
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.06737233028287096,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0013379489246377642
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.09693941896958716,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0017781152777598497
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.07283735339621857,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013034613782005062
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 0.4367609232393782,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.023336487096873623
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ffa7fb4413dbc8d13def39f79af575dde223b80a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.10962094154854289,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001521175658021088
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.11614995439958446,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0016222282716261732
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.09906222042576335,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012277637555703933
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.005375616655817727,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00032065018661247955
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.005997760028548002,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00040011451896884595
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.004986359493861402,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0002877838903421649
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.08788884421952135,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011845230529361117
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.0930440125082319,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0012664554630627011
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.07875322436499038,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.000899325603615052
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.10514951471452734,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001446840001157129
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.11139163899995776,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0015440273459916564
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.09495364214558669,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001162653836627394
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 0.33862715230568985,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.02103608050658023
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ef299d1e10050912ccd7c14000177725ed852173
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.11008815005619565,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015367268621741792
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.12364976087148276,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0017477601358293203
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.10124369006753632,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012431775944759696
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.006329778123956924,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0003579493269657872
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.007946063514698743,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00048230591130947426
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.00600392083872442,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00031876702476632513
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.08810707825607618,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011970905402063467
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.0995681437810585,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.001372354219556691
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.08055079492734035,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0009086151054529219
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.10550757799818274,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0014666803119605407
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.11816997810521175,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016494951956388857
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.09678091628110796,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001168106008335189
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 0.4064700344068096,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.041071462910743026
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..488dddbd4baf87159932ad696f5615b847864a85
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.094117032806795,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017305123351559388
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.1061886960885055,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0020438338074463868
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.08498062296250532,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0014212404882767398
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.006623081561306862,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0003792072547215317
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.00957912154003632,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0006732767370116441
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.006494830300354191,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0003529124882286109
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.0760422274749195,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0013874920699279642
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.08570210788487924,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0016287020168935214
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.0677917896388117,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001061516966060374
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.08992646673675982,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016454052932894198
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.10125000826231054,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001933208130218866
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.08096977315853292,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013335904643873365
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 0.5087314599386397,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.061212421752417735
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..79bcb561669db9061d152163bfcfc952c9ef2464
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.033017953252287624,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001455074891420513
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.035407839416534886,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0015340457724425114
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.027959665353093514,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0011329907447503772
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.0028089869136647076,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00035883040231905175
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.0033990005640009368,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00043264989107393107
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.002417994702779381,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0002673599459103152
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.027079006620999064,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012047423398949
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.02913572117563057,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0012482614882508539
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.022633019178426736,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008874765902663727
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.031138458647991887,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0013669267038162584
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.03352557536851953,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0014450564773056973
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.02640979987744244,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0010615456146286408
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 0.05513061350682931,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.01363489004556789
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..8b3eeea09125313933b35d55dec0148865ef8263
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_rephrase_en_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_precision": 0.004907621925869614,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0005763406359517721
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_recall": 0.0053768195418473184,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0006637161154816707
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge1_fmeasure": 0.0042516054308208645,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.00047637308191102623
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_precision": 0.00047314584408024164,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00011906110853937211
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_recall": 0.0004964345162201845,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00016880696249774884
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rouge2_fmeasure": 0.00036529417385801794,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 9.087182991829831e-05
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_precision": 0.0040961836397586295,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.00048820210285777777
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_recall": 0.004329727791926256,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0005262040146135757
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeL_fmeasure": 0.00343786505841611,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0003771726212323476
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_precision": 0.004702148837329425,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.000555612812542979
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_recall": 0.005058029292746475,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0006171534300008613
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "rougeLsum_fmeasure": 0.004022188513290089,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.00044739410245404963
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "rephrase_en",
+      "bleu": 1.9107654017640184e-11,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 1.3525115607349213e-09
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..c9954cf81a9205401a08fabba29cb07b18f34532
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.04418667472732478,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0012197029509449378
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.07215894137790155,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0017665079751954936
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.050630622435102954,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012740642440480783
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.0033950280387810075,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0003033243161776345
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.005381920499729423,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00048448492307215835
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.0038458659144674466,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0003374984459731064
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.03927568004870594,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0010304820118418286
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.0651799283564036,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0015341625257454653
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.045212023379810784,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0010775987760050663
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.04166366640338202,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001132597373996381
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.06842472788539408,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016398413239512239
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.047792067497451136,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001175048993772459
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 0.2645947287972745,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.021907771840493258
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d557ee84a509b943575af9a7bb95257f7d1f7818
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.11065265004766811,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0014995222918311947
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.10967982839934531,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0014540559708165963
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.09711652466721561,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.00114856823101296
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.004476358135369021,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00028847561060159696
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.0044039325000597256,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0002972809876903113
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.0038867114952000873,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00023684798185580298
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.08933483124530466,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011884320054885904
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.08853836595721083,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0011479891985354547
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.07784452056813264,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008619620021596947
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.10659695950864206,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0014326181370186257
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.10579025460545817,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001398522609494789
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.09355863188288549,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0010951306163491682
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 0.26189100567433143,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.04164591416025924
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..39238025d5b51e75c92fac9354377267f3c2749a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.11362728657731791,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015714871657739951
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.11607779218894242,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.001575602946003843
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.10049145155089874,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0011946171315062168
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.005836029460495955,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00036095981618922905
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.0062244104824258515,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0004249628756393917
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.005194947294074261,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00030474043998879574
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.09152414632340847,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012419260737023521
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.09395918617125179,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0012571326025089399
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.08052988976365942,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008954855289212289
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.10951406198494446,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001514165066763457
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.11185509288980913,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001509586047459457
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.09676724655543643,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0011401063414987313
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 0.382732965383335,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.041457097975708616
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..979d473799f8e2ee9229676534c334a3f1a7b28d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.0955940850522753,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0018130428010804185
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.09383021551741794,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0017574878214917625
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.08043785817220254,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0013311625960759211
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.005894324763379858,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0005451846101275158
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.006111381952443814,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0004988935962431043
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.00471973086085565,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00031285101854069914
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.07845129854611516,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001504183246143632
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.07671122424431512,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0014337379247573415
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.06517286449766353,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0010214075492220697
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.09173213439305535,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0017274967484957875
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.08992169460486722,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016733690010191946
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.07705416712954397,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0012575955853530694
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 0.3919914059035021,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.06209565085248539
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..80e57ffb356f52b1a5334cc0830d2c113173bba9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.031069492435776818,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0013777037826483293
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.02923924720251318,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.001332321685582368
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.024625870173260112,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0010164821198364775
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.0022741707476197034,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0002784957287667364
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.002541556245852753,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00034331230832384933
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.0018715335775846497,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0002104568234283972
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.026118959440308825,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011609305606255281
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.024492403731580736,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0011008059304958085
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.020411284941508113,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0008173009244281204
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.0295178758499726,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0013119054474959343
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.027611772675785972,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0012447336136434688
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.02329804778849391,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0009534761646565726
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 0.01607270818304334,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.0034624809654178012
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..79b04e5103f2d296cf180f498888dade871ed501
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_summarize_above_en_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_precision": 0.0046970714312177565,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0005792165335246764
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_recall": 0.004114523129900782,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0005483319592202135
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge1_fmeasure": 0.003521129990818492,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0004036686379505131
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_precision": 0.0002427708680341301,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 7.969113434382603e-05
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_recall": 0.00038997340658384743,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00018348649094998182
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rouge2_fmeasure": 0.00020048952920264672,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 6.865942316355645e-05
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_precision": 0.0041148184629156,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0005057838120170794
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_recall": 0.003574300499782594,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00047502389608598274
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeL_fmeasure": 0.00304301527398751,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.00034135496557363107
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_precision": 0.004523141975654186,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0005607431680479185
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_recall": 0.00392102854568574,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0005176638248032069
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "rougeLsum_fmeasure": 0.0033595582196200113,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0003831160597743695
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "summarize_above_en",
+      "bleu": 4.950641247853647e-16,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 3.846396789441964e-14
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ad545984f2c47b676840dced9934e4d0b01e20f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.08475422777270436,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017956983368717673
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.136466539852913,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0026047032098612134
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.09664038969174059,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018523781781760917
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.013532165780802595,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0005500871126004836
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.023126949071911686,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.000984728105845212
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.015671748077472996,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006102024889325685
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.07093659031967181,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001414812786895555
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.11686852788639902,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0021776574996213755
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08141292643254222,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0014662345345116555
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.07902371474904281,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001655136022979042
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.12774835789500472,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0024261371686350093
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.09028162427912649,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017170249037180556
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.9033237981203045,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.06093048704966743
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..46e86b0b59cdc3c920427ede93e3ef52129a128b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.11341569440193765,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015875420885948467
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.11847927588394622,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0017387628767267596
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.10140029000732263,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012698513145956132
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.006128471241361133,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0003870105678849676
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.0076069170914782195,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0005734677696249654
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.005755944338368179,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00034584001151602705
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.09114212196429179,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012300275412546436
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.09521970565929594,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0013695362241421447
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08090174235356164,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0009344918578662129
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.10896112722491796,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015092137229918339
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.11363652894302366,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016407183665521966
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.09730799225374498,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0011957026786911643
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.48568473425226133,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08021116263112237
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..46be08418bb85d241601ea121bb34527e8ccb2f0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.11925916564183478,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017880379896240603
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.13943695708117368,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002116433411707032
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.11151220891599137,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0014668379581459778
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.010505863803598392,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0005738646736412342
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.014684578264016469,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0007964172087895143
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.010504603460980755,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0004959353821993032
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.09588263877954578,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0013996502096768437
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.11375527509576841,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.001731363662015009
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08956417926961593,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0011103097178602006
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.11339532279740656,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016872131131246602
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.1323364741919991,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0019843253951356537
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.10583744233386468,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013667666533331855
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.7735409292280904,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.04682416335531492
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..5080e142e8a18262dd953c1eef99b8bb55ec78e1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.10733987691391439,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0020889839913980418
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.12476072254733013,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002308247813491843
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.09780113881018691,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0016273262648512452
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.012191906060064263,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007006501461727509
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.01532295103209109,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0008648292695711531
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.01096682747964428,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0005407448750885603
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.08876565747817264,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0017029372366386352
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.10463111270816423,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019111151657166508
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08074375845019817,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0012686172017943162
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.10123055336456097,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0019729409528934246
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.11748223446367839,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021566918011451134
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.09202413237033855,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015127002889302342
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.8642314685339636,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.07105912584286404
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..68b71dc711c2449599426660d966a613b02f42c6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.04130438911538399,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0018399622962315205
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.04523262499737334,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0018427581724112859
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.03531682035958441,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0013697458060726072
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.0069981887164715815,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008066183569971981
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.00693668855148366,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0006076952741663856
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.005041922188233573,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00039035553600462724
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.03459558732097317,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0015673695864144438
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.037976749671683144,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0015299014996893944
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.029228194708275798,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001099845646230128
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.038549446328862175,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0017184540964501843
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.041863381072326546,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016845634789296187
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.032707886197975086,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001248884867759778
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.11591220265958264,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.018810551681001538
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ce3b75bccbc0281ea3416a7237654f351a1f37f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_tldr_en_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.00840179041155278,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0009319251831963879
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.007933117544594833,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0008986804343115746
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.00646829859544361,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0006749836942920999
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.001300602211987841,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0002505531879453807
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.0015037014195862734,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00033970791079948175
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.0010361138600270753,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00019163061331141026
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.0069081946455306804,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.000767045855376856
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.006517809586183419,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0007338789355087525
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.0052628072217658275,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0005418137104031803
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.007887486215836917,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0008833842095846506
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.007316433171910116,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0008198190943819032
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.006026061380800861,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0006299887360163144
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 6.457241109219731e-10,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 4.3951639243907024e-09
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..34bf43ae4ed03b00e59c2c2e751d9f9e430620af
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.09246228962154374,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001943807421175789
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.13805949306072055,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002409786846441639
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.10023372190565902,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017262119489058103
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.012545852280114898,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0005482585896654373
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.021241128219617465,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.001043761840842228
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.014303497306763413,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006015388106615314
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.07837678044248741,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001609888637751305
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.11893781436000582,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019490781148502943
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.08504732396245059,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001313235354820681
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.08436461190041952,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0017072730429511782
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.12779895161379792,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0022186936931219443
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.09216848394827927,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001563789121056033
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 0.745448160166905,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.047547732193019436
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..666e687c2fcf227bd953246c76158fa8d95da007
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.10990140516594758,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015351949382576508
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.11791561392844088,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0016685835887549996
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.09978220637884393,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012449757420128587
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.005589121653905529,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0003414693806880958
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.006477748246242698,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0004490780641964607
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.0052645090649109895,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00031778114128576976
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.08809126672816353,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011930762685352886
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.0946530891608615,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0013251798935214189
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.07936011949636798,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0009144164542617104
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.10565199584207204,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0014615877820162168
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.11343373068042586,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0015989418107090036
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.09589920116880182,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0011829464434838014
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 0.36679033215823664,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.029277836361420107
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7af10fb1cabfd9579b183ba23e37970bdd81f41
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.11013619429611844,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001546664571183218
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.1232803535134769,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0017517550316112635
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.1016546124922999,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012651922442427001
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.006234351643431517,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00034125509784363245
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.007962049327295925,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0005161503037287333
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.006048464517830181,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00032306729820669704
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.0875404512463235,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.00118479594104557
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.0982995279262905,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0013504657871243205
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.08023398785251074,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0009106005118733982
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.10557342164225539,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0014719765283516568
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.1177632536098584,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016421313291817262
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.09722400124042876,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0011899218963608768
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 0.3820141559457903,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.03590316359310684
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0dd37d5d952961fb2a9496265c2ca2161019e5d3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.08899396863415189,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.001777790692308317
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.09692319541629159,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0019946027284699383
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.0785984819107557,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0014372932969101489
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.005984354455890461,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00040684067708898005
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.008510439649050199,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0006316823704492148
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.0058753493744521336,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00036684689106018846
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.07217146798975321,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014091242615705865
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.07831148959974298,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0015814306656436506
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.06287166017244038,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0010782265186488856
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.08495497287249752,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016708456442186747
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.09237830202855782,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0018778739354038604
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.07485251192796798,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013431259530301976
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 0.5176377216004082,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.06866328234036077
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f772e57d94214e88b55dae0949e8e68ce21a2ac2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.024653052383932526,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0012369651915557867
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.026978552912746406,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.001367418485604499
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.021150352711754538,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0010009476386843301
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.0020847909583644186,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00028781597725583064
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.0025592724240508426,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00032606918523288823
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.0018364476816945278,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00021456601406933008
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.019887895435798185,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0009981749275954795
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.02207509587363469,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0011126244037750546
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.01695587958257084,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0007821077415757568
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.023148124955408137,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001155778717437976
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.025400258275254814,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0012784306268269182
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.019854394029124416,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0009334487521584576
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 0.013454379756596573,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.004377302392144288
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..194ed8b563a0e1618ca40c794626be1fbd08b12c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_GEM-wiki_lingua_en_write_abstract_en_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_precision": 0.002594820378060942,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.00041260070427087084
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_recall": 0.00315946115980561,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0005463787261087583
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge1_fmeasure": 0.0023404225914027293,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.00035897358936479027
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_precision": 0.00020295754731072598,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 6.30448872924687e-05
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_recall": 0.0004422474374560793,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00017450355164885773
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rouge2_fmeasure": 0.0002340501971901335,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 7.27640076429073e-05
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_precision": 0.002059754228694586,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0003240033862130502
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_recall": 0.002522670751072346,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00043340282734999423
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeL_fmeasure": 0.0018292868173016377,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0002697379131437824
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_precision": 0.002476388701828487,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0003949847874533559
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_recall": 0.0029610416476689777,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0005018145536321888
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "rougeLsum_fmeasure": 0.0022043625397931894,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.00033340306926608464
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "write_abstract_en",
+      "bleu": 7.304724850661707e-22,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 4.651649551405445e-19
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..58d1da9e12e84c847a54b8b72dc1221556a58e23
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.323,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014794927843348633
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014910846164229857
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1725ce7ba0a693959fd268a3002a03e6b4b3b61
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.33,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014876872027456732
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.326,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014830507204541037
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d4b932d2482b9ce130c656cdf7eab80fac93f2a1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.354,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015129868238451772
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.35,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015090650341444236
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4401066d0da6ac12e6a262fdde4d1c1331a21748
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.347,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015060472031706618
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.346,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015050266127564436
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..85e46a663a8f789f355ec9589495634774eb5756
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014955087918653593
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.345,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015039986742055235
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ec71be76ae7814836764ac7611303b199c76299
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_GPT-3-style_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.344,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015029633724408948
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.01491084616422987
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..8df1f005d002d6ecafdd651cd93a1e501a2e49b3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.321,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014770821817934654
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.334,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014922019523732967
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..5400bb4821ae4877b7e1bf88d53f69abea35463f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014910846164229863
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014910846164229863
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a2e37c3695af85e0a337893eb19bee1d3235805
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.338,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014965960710224485
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014944140233795021
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..274f3d1650c9f124c463ada2b29d19413dc3c20c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.373,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015300493622922814
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.361,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015195720118175124
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f10e077326f7f3e212691f44c2cc6c389969eb7a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.346,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015050266127564446
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.342,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015008706182121728
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..de0bb89071c85ed7e4a27626f8002de11e4b64e0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_MNLI-crowdsource_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.338,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014965960710224485
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.01486539538592836
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e62cb73043b9c568a7ca767677484b8c1ac48cbf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.321,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.01477082181793465
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014955087918653603
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..82e436698600e229d4e4fb8ca7a5aa7317244e75
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.33,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014876872027456732
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014910846164229863
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7fab316f42d1b00a5fa67f1dfd0615ec018de69
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.342,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.01500870618212173
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.343,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015019206922356953
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..af6c0d1a68f7ecb338b3e7520632297be4026c1d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.349,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.0150806639915631
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.36,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015186527932040115
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b821f9c687532324138c7b03c5b510f08577e333
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.341,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014998131348402709
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014955087918653602
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..996fc1bd196961ef7fb3ca1489efe7a5538a389b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_can-we-infer_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc": 0.332,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.01489959724281149
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.33,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014876872027456736
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5ec4875e748a67a209d19b216acc67d07fb868a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.318,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014734079309311901
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014910846164229857
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..064a803f3cc40a63990f1f3a58bd5c80355b3830
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014910846164229863
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014910846164229863
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..34769cb27ef07e78a1935f401dd9c540c6979323
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014944140233795021
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014955087918653603
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a48a53596a793a72eb2222db02f178d54c5e7be
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.339,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014976758771620342
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.338,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.01496596071022448
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..3f3c5797afe0c202c47dee19a8cef5e0fdf5bcae
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.319,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014746404865473486
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.326,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014830507204541044
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..991f818c76d09df854ccd35ea76cc36016efc206
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_guaranteed-possible-impossible_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.324,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.01480686473373886
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.324,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.01480686473373886
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..b728a89ddf48d799576fdeaebb796aa234efcede
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.32,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014758652303574883
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014955087918653596
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4570f2f006a81f6b0f67b75a35d513e36ed761ee
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.327,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014842213153411242
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.328,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014853842487270336
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..05d3346873e97d8a7b052afc0d4383b4f4ffb27b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.346,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015050266127564431
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.353,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015120172605483706
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..c186d883ee8eab604692bd8369d40c59f7003986
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.351,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015100563798316403
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.35,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015090650341444236
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..896e7608830803dfb8ccf86069d7c20c81ac770d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.342,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.015008706182121726
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.347,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.015060472031706618
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7c4e8ce1e50d9fc0f83c91d7211c9d90e08d50e6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r1_justified-in-saying_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc": 0.339,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_stderr": 0.014976758771620337
+    },
+    {
+      "task_name": "anli_r1",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.321,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 1,
+      "acc_norm_stderr": 0.014770821817934652
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a4208d3a585fa351251f228cbedb858e79a1f52
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.32,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014758652303574888
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.01491084616422987
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cccf80bc38f88a797bee5810baf77e593ac7bec
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.318,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014734079309311901
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014696631960792508
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3f09b0648edc97593cbafdaede9d9312cbd4324
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014865395385928373
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.324,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014806864733738864
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..55cb2bad98e7c8ab6d99020719be1f53abf8d8b6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014818724459095524
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.317,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014721675438880217
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..9495cca5fc7f94beee4cb643cf29b7af4f49a933
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014944140233795021
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.322,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014782913600996681
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..0b9148c816df99634395a72ccd215d222bce4612
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_GPT-3-style_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014910846164229871
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.33,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.01487687202745673
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..44d13444d6b50ac9c541a977551ea9ac7d48a0d7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01494414023379502
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.344,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.015029633724408941
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e1f186faa13e5a728037b0e3b60a78a32b83a3a7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014696631960792506
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014696631960792506
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6306f67a4f1cca7997a7b33ea9548fdee91d24c6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.321,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014770821817934647
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.326,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014830507204541033
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..6e519ca4125185f240f2322fd55d601788e23a3a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014818724459095524
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014865395385928362
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..2582e4d8b02c13cde2d7d51d229a814de00a9b24
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014818724459095524
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.328,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014853842487270334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..29cc0d48854a289a3bcfd05081d45e9dee7d8bcf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_MNLI-crowdsource_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.303,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01453968371053525
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.317,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014721675438880226
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..91b20bb7ef6ea74bd361a932c359d5d1410044ed
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01494414023379502
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.332,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014899597242811488
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d81b0536041fb55a8590293068d1d31338e3ac0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.318,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014734079309311901
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.318,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014734079309311901
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..536980cb85e15f8d0d1873ec9c8b34963b645eb6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.324,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01480686473373886
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.321,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014770821817934645
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..62074c506cd6a0ab7a9fb5b460ec82606d16f481
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014955087918653596
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.341,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.01499813134840271
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c248fc44228fd389969cac3e1d937891de91f8d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.328,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01485384248727033
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.01494414023379502
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e22da91f38694047fb3226698422301c7ab98d5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_can-we-infer_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc": 0.336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014944140233795021
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014818724459095524
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..13da961bec760c4fafd681f1a1ff9c1d8ddb942f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.348,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.015070604603768408
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014865395385928352
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c46747d830803f4da314ee120c699501a878cde5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014696631960792506
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014696631960792506
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c69b13bbd1fe77b82b2a1ac207e8acc46eb39706
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.32,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01475865230357489
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.317,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014721675438880213
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5860fd24596e97b7aec296e45d9ee5bbd13f27e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.326,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014830507204541038
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.328,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014853842487270334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..442c2b54325830b0205d1436119eb1ff3f15784a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014818724459095524
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.32,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014758652303574883
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..20b886209350404e82b4d6ec44a70b55afa90716
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_guaranteed-possible-impossible_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.317,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014721675438880224
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.314,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014683991951087964
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..ed925daaba53389e14f17e9b2010dd506a9b2684
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.311,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014645596385722694
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014865395385928367
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c6592078d31a76c1d705e3323fef99688c45219
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.32,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014758652303574886
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.319,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014746404865473479
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..584b3571b96e825d4f3c8537fe2ecb718e63a59f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.311,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014645596385722692
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.322,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014782913600996685
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d8acfbf5d1aab583317f3f8c4d494ad0cb37526
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014910846164229868
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.341,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.01499813134840271
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..11c51955abfb62369d56e6a96d160943340670fa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.329,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.01486539538592837
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.323,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014794927843348633
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5dad72a4a0dd40c00b424746e286019571eca786
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r2_justified-in-saying_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc": 0.328,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_stderr": 0.014853842487270336
+    },
+    {
+      "task_name": "anli_r2",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.337,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 2,
+      "acc_norm_stderr": 0.014955087918653596
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..dd79c73640ab4f2a4be5bf8a24175fc88efc98b5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.30583333333333335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013306526255831147
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.01363087184382147
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8a2e1f035c9e6985aa1c9a93c7773a7bcdc388cb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.3408333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013688600793296932
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.3333333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013613950010225608
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b66e437c545d6f336c86d4b6b502768842a0d482
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.3416666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013696658778002519
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.33416666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013622434813136767
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f7eb7a86bc3eadee2dab36b211b1e6436f70e069
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.35333333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01380457216231493
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.3575,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013840921245257787
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..46a2bc01c8e2228dc46137a0c800753341bc0576
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.3416666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013696658778002519
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.3308333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013588208070708995
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..666d578ff09be9db16231f1c1a54d2255b39eaa7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_GPT-3-style_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.3233333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013508372867300231
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.315,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013415009084004859
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..28f1a095e1665dbc97bd1a2255b2a171b8da9353
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013630871843821477
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.34,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013680495725767787
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a5d709a381e069eec0a31ed81e5f080403cbde4d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.33666666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013647602942406393
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.33666666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013647602942406393
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..4b2627a1ebd97c6100759cfd8ac254e5d07e825b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.3433333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01371263383046586
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.3283333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013562032919529015
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..36c6e5836b74142878ae76178f0040e059eac70d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.325,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013526454480351023
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.32166666666666666,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013490095282989521
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0615a9e71b201e2c16b12660e6cddd5153512a0c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.31416666666666665,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013405399314984101
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.30833333333333335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013336721143136467
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c1896e398fa14079eb154f8fc2d5922ad156252
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_MNLI-crowdsource_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.30916666666666665,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013346684134591945
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.31166666666666665,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013376268790982112
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..24ef9e914b4b306aee817c7e2a9d003a542f2829
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.3433333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01371263383046586
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.3441666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013720551062295756
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a4d28ff7cce1511970e17f050377873d287cd23
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.3441666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013720551062295756
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.3425,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013704669762934725
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..473184e95de11d3a71323c5863b8ee48bae5179f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01363087184382147
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.3283333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013562032919529019
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7a7cde86f67c23bf65b94d985b7914fdeef11144
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.3375,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013655897185463667
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.3425,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013704669762934734
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f820e129379ce1b1e6e938cb57731d97ecc689c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.3358333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01363926119093287
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.32666666666666666,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013544340907003665
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..451dce108e0ce2a632f707b1ebdc4226222eeee0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_can-we-infer_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc": 0.3258333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013535422043417471
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "can we infer",
+      "acc_norm": 0.33166666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013596836729485163
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb97b168aa3963af004c5226d2ded3f3ad99d64c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.32166666666666666,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013490095282989521
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.32916666666666666,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.01357080625843362
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..90b0c9a61c706c47d0eac05c592107374cde8f13
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.33666666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013647602942406393
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.33666666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013647602942406393
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b809e345c7f2737a33e20c7099a26afacdfae4ff
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.30916666666666665,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013346684134591941
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.30666666666666664,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013316642319070695
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..dcd16e8bc4fd9cf1a51f91944abef7e8c0eec594
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.3125,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013386029277441229
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.3125,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013386029277441229
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8a848fc86be524a896ce9f6e5fff913af7c2f506
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.31166666666666665,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013376268790982112
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.30583333333333335,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013306526255831164
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dd4dfbc1ae34a4188d923cc610225c8218daeea
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_guaranteed-possible-impossible_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.32416666666666666,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013517438120881629
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc_norm": 0.32083333333333336,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013480882752851555
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0618272b2ae8d9c2a18265010c4779b2e131bff
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.35,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013774667009018552
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.3308333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013588208070709002
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..9a029d6dd98568e640678d862fae3f3879b69501
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.3466666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013744022550571949
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.33916666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013672343491681819
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c3b17bbcdc9346923627c1f5fe369a97a416cac0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.3275,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013553211167251953
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.33166666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.01359683672948516
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f5f20d2c5be89a3a3c8d485ed6df49d5a20b050
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.3458333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013736245342311012
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.3425,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013704669762934734
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..245389a12ee2ecf1b1734e4de7f2faff6039a4b7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.33416666666666667,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.013622434813136764
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.3308333333333333,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.013588208070708986
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f3f5c73c748c5eed7d4103a70be0ec2d260777c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_anli_r3_justified-in-saying_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc": 0.3275,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_stderr": 0.01355321116725195
+    },
+    {
+      "task_name": "anli_r3",
+      "prompt_name": "justified in saying",
+      "acc_norm": 0.3275,
+      "dataset_path": "anli",
+      "dataset_name": null,
+      "subset": 3,
+      "acc_norm_stderr": 0.01355321116725195
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0debbec77867ea728c0321e3e4a9cc49367c75d9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.23293515358361774,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012352507042617396
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.23293515358361774,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012352507042617396
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..75e9e8d57df3c56bb5253fa21a886f335c160346
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.24061433447098976,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012491468532390573
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.24061433447098976,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012491468532390573
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..81c171241df195a968cb5b424c3058445a2998e4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.2354948805460751,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012399451855004755
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.2354948805460751,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012399451855004755
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..2957c5f7af4292510dbf72faf0f85b87d279ec55
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.23890784982935154,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012461071376316616
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.23890784982935154,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012461071376316616
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0aca7ed7eafe4a3cdc5ac48df214139154b17de8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.2363481228668942,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012414960524301842
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.2363481228668942,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012414960524301842
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a398d9c86adeed57ec165b560a78094d2e65ed6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_heres_a_problem_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.23378839590443687,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012368225378507161
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.23378839590443687,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012368225378507161
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..935198658eb0e650d95126d134abba30f8bec7be
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.20563139931740615,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011810745260742578
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.25426621160409557,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012724999945157741
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3790a3777515f905485b3b5a4334fb0877c3ee0a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.19795221843003413,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011643990971573401
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.26023890784982934,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.01282193022511256
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c67b99cad35d574204afb28a4960adbf454980b2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.20477815699658702,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011792544338513402
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.2593856655290102,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012808273573927094
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..8cc6e3e28e200013bc401f458541dbf4592e9902
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.20648464163822525,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011828865619002316
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.25597269624573377,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.01275301324124452
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8667bbe029e9d59152e5a158e0c66e00f0fee259
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.2030716723549488,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011755899303705583
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.2568259385665529,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.0127669237941168
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..f12f937a1426fa83da5bad4a53b4a4becf3163f5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_i_am_hesitating_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.19965870307167236,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011681625756888692
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.26023890784982934,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012821930225112556
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..28c26f4d98613cac7451b9704c6cf90b426788e6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2235494880546075,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.01217489663120261
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.28071672354948807,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.013131238126975586
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..12c562f8b75031ecba7d8349dc1c41febfd64de9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2175767918088737,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012057262020972497
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2858361774744027,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.013203196088537369
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa9bdcae31a59de4c3333bf72b9287d44e2da64d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2158703071672355,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.01202297536003068
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2687713310580205,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012955065963710696
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..6bcb1a36b68cb96af8ec357ae422565a37f1d724
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.21928327645051193,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012091245787615725
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.25597269624573377,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012753013241244532
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..343458a720846eb555e4df8f06559665ad545abd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2150170648464164,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012005717634133604
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2568259385665529,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012766923794116801
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2500ab8fe9a28722a81f2f9d07fb853d6ae2bc99
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_multiple_choice_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.23122866894197952,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.01232085883477228
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.26109215017064846,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012835523909473845
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..480ea0de2176d366aac1719014b1f4e189d83c69
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.23464163822525597,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012383873560768675
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.23464163822525597,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012383873560768675
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b422a5053c8d029345f9a1c3cec2da7b07780215
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.24061433447098976,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012491468532390571
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.24061433447098976,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012491468532390571
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..042187764a9bb9032271f42e7112bf0b0032f54f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.2363481228668942,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012414960524301839
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.2363481228668942,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012414960524301839
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..a6e7654bd3b1361c375b8e80cccabe761f594826
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.23293515358361774,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012352507042617393
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.23293515358361774,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012352507042617393
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..3aaf32a6e525d3d2228418215ccbc17b3fe6cf60
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.23464163822525597,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012383873560768675
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.23464163822525597,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012383873560768675
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..11d68f13df74f54d98e381e9d34e3e90f6b0de5d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_pick_the_most_correct_option_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.22866894197952217,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.012272853582540813
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.22866894197952217,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012272853582540813
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..927eba3afea3142e0956eb8c55e72cfa6e76bbce
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.2030716723549488,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011755899303705582
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2568259385665529,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012766923794116801
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c77337dd84c0aad45c3a924f9c85e4ecad0b184
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.20051194539249148,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011700318050499375
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2627986348122867,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012862523175351333
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d50b2057704239c1c1107791435edf377ca61615
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.2022184300341297,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011737454431872104
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.26023890784982934,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.01282193022511255
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..1252e9a828fa2719003a3a9b2e578e13fbacabdc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.19795221843003413,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011643990971573398
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2593856655290102,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012808273573927092
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..175c79b381c8daa384465c8f09cb0a9e4481526a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.20648464163822525,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011828865619002316
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2568259385665529,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012766923794116798
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..e0e67204bf2d8b160853bdc5837b300bf787f3a9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_challenge_qa_options_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.2098976109215017,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_stderr": 0.011900548748047454
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.26023890784982934,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Challenge",
+      "subset": null,
+      "acc_norm_stderr": 0.012821930225112552
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f484d3543f2da99c4c810aced5927b5ead75176
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.24957912457912457,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008880241465504347
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.24957912457912457,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008880241465504347
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa28ad8b5837795226aab33953c06da81aa47d84
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.2474747474747475,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.00885511441483471
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.2474747474747475,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00885511441483471
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..06325bb7c8b4bae6c4e5236a4a5418c09151da4c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.2558922558922559,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008953950243013991
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.2558922558922559,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008953950243013991
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3bfdcb6c92f92d292defda5ba91157c1b500f8ff
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.255050505050505,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008944265906130709
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.255050505050505,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008944265906130709
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac110bde75bec8e679768cf8d7f6e8db1118c225
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.24452861952861954,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008819461106822598
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.24452861952861954,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008819461106822598
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7305b3d300315f32efead7cce426b67b8f1d4b9d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_heres_a_problem_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc": 0.26346801346801346,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009039157374497713
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "heres_a_problem",
+      "acc_norm": 0.26346801346801346,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009039157374497713
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2c5886a478bab9292b46870747b6d0d1e084a060
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.2895622895622896,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009306838912173909
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.27525252525252525,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009164888895174743
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7805eaa7e3bb41721c8b637547bd7a4b6feab3fe
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.29124579124579125,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009322788837938861
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.27735690235690236,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009186490105111902
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c3a5ff39cdd3547df7d8360af75422bbf202bd5d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.29797979797979796,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.00938504606669487
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.27104377104377103,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.0091209197417606
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..95f61720ee5c6f45e4d9d8cc8f9e7c45cf890174
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.28703703703703703,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009282621598983068
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.273989898989899,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009151805901544028
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..146d58e6eedac535d334a877b48b8a5a6e4780f3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.29335016835016836,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009342508331708563
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.273989898989899,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009151805901544024
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff56832e4d6a66b87e49455adc7429e7937d8a29
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_i_am_hesitating_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.2908249158249158,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009318815921176647
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.2760942760942761,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00917355987383526
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0f1ada0c0049b62923de01c59a85c9b85f1ad2d0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2895622895622896,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009306838912173902
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2706228956228956,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009116466166403825
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e8a4d6f25b4f6ad67802788a62a0720797ff9df
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.28535353535353536,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009266280584997753
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2735690235690236,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009147424438490741
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..3a6279f63541911c7f9dbaea9272e80ca64df0c0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.28324915824915825,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009245632200075456
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2786195286195286,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009199329195026348
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..81bba1ea012d555bb1fd25a7dc35728a85ff5476
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.29292929292929293,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009338583737393607
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2857744107744108,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009270380606981212
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..adc77995fe672e8a4111ba045ca9c36d55a90914
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.2946127946127946,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009354224395837087
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.2828282828282828,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00924147277532823
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c9da5b95f7b16bae259e7270e4e21671346c090a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_multiple_choice_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc": 0.29124579124579125,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009322788837938863
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "multiple_choice",
+      "acc_norm": 0.27946127946127947,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00920783814259724
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4034b1e24dfc6c3967f59af113c1cae29230553e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.24831649831649832,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008865199020660961
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.24831649831649832,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008865199020660961
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..095f94b3030b1f6df5be761ecf53fe924e0b79eb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.24873737373737373,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008870224411653797
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.24873737373737373,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008870224411653797
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..f01435a87de19aa0920c25b5c7cf9d0e22b2dda5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.2474747474747475,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.00885511441483471
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.2474747474747475,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00885511441483471
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..337fbff41e54b2c726c3199ce0fc488b123803c7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.2542087542087542,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008934537681141544
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.2542087542087542,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008934537681141544
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..94821f25bd16852497a85a9dfec202a187e6134a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.24873737373737373,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.008870224411653797
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.24873737373737373,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.008870224411653797
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..12a39e714a0823bc2dba69cc180f79595a9ab047
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_pick_the_most_correct_option_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc": 0.2622053872053872,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.00902519799172483
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "pick_the_most_correct_option",
+      "acc_norm": 0.2622053872053872,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.00902519799172483
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fdd26933020cbeba22329264a8893564f43fe54
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.29208754208754206,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009330705616569072
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2756734006734007,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009169229476542569
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b8df8682d5c12cf697867024c8ff2619d2eca71f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.2975589225589226,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009381226721815539
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2807239057239057,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009220526174711356
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e52cd4608405fb2e9b98ebc9899477f305222b9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.29545454545454547,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009361987126556458
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2824074074074074,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009237303403479327
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d535126f15d2a126fcb72f3f7a2778337899d5ac
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.29124579124579125,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009322788837938852
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2828282828282828,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009241472775328231
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..5050822f4cad03cfe3138a0b5351077c97e1806c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.3005050505050505,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.009407763090599316
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2828282828282828,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009241472775328228
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d85a1016c285cc21fe7b3f510e7aa25db03b2196
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_arc_easy_qa_options_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc": 0.29545454545454547,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_stderr": 0.00936198712655646
+    },
+    {
+      "task_name": "arc_easy",
+      "prompt_name": "qa_options",
+      "acc_norm": 0.2824074074074074,
+      "dataset_path": "ai2_arc",
+      "dataset_name": "ARC-Easy",
+      "subset": null,
+      "acc_norm_stderr": 0.009237303403479332
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e2b97171f7a9078abb200354d797ed975161f765
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.422,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009018450207660421
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.6236666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008846558976258922
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..0b338cc7f2997decac7033ec2203467b5c32801b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.539,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009102414587191052
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.5456666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009092070195065414
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec32cfe3a05eb9df0e3ddce5e8e6033fb8ae53df
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.5253333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.00911850443412263
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.578,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009018450207660424
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ce1ab68048e872fe90872d7368b39851f96b166
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.5316666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009111901539047272
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009012606487132152
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..99bee9d03638ebe154a13686e72ab646d1340e80
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.5383333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009103358843448796
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.5776666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009019409415904176
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..613657da23f6cfbbaec0ab0bb8d19887cb92f437
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_GPT-3-Style_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc": 0.5336666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009109510391803648
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "GPT-3 Style",
+      "acc_norm": 0.5676666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009046234144187917
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6acaa60056802e6c9124ee5cfdb1126930838faa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.6236666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008846558976258922
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.543,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.00909640486825282
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..137e2d06643cc6c9f4cedea62d106a6b11bb8e79
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.5426666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009096928229880423
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.5406666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009099982269204863
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..fbf41a80cb20a538a56334e58e6b38cdd61e494f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.5673333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.00904706345689798
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.5543333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009076164124491365
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..2e19d74ccc6d5f1c86939ce217b22d8833a6bf45
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.5646666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009053547904033172
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.5616666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009060524549335622
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..df321949d5af568621ee166bf0a3a7e36cf879ab
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.5673333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009047063456897982
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.5583333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009067881941319685
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..569f6bf8d8568116a186b45fff943556de336933
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_after_reading_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc": 0.5746666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009027853030468729
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "after_reading",
+      "acc_norm": 0.565,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009052751926300883
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..43d25f551ca390fdd68b6db76bc9b562d637ae9c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.6236666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008846558976258922
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.3983333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008939496695192017
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac08ef2b1f7bbb4b436d54218ea887abeee2a8a5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.546,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009091509877386517
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.5436666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009095345834327865
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b4d4419fac0683ac9226a9218168c940abee946f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.5633333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009056690207178121
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.5506666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009083233528874796
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..f96ae7fe1861841c2d042fe0ef9854d056dca28e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.5626666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009058236409215862
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.5496666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009085074954912701
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..7448e70fb5b056676a8290e67faab14d787dc8b3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.5706666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009038582451449423
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.5546666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009075496684215473
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..81ad791db99af3fc7ebcb372d8c5ea332cbcd2a3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_exercise_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc": 0.5703333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.00903944935393088
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "exercise",
+      "acc_norm": 0.561,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009062029213030573
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d769728ff890eb41061c06eaba8ff61cd8ad552b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.6226666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008851200156534391
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.441,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009066443632063164
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..18a07606cc70fba6a5263e2eed2a7bf5c0405888
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.5406666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009099982269204863
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.541,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009099483512819305
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..31237584fab9834a727e4fb38554a4a482541dc0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.545,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009093178503605508
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.526,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009117878695396636
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..b70da623d186bc093ea86d8f3958a8c6ebbb8eba
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.5583333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009067881941319678
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.546,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009091509877386517
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..c82ffbdfef8bf665e268c1cbcc83d0d0429f9895
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.551,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.009082611478924382
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.5406666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.00909998226920486
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d99786acb72d72c036bae34ddc419e42525fce64
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_valid_binary_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc": 0.5626666666666666,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.00905823640921586
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "valid_binary",
+      "acc_norm": 0.5513333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.009081985306932099
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f4939261ee80d77b49f2bb5e89d8a5ec9725336
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.6243333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008843442555522142
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.6236666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008846558976258922
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c9293fdbbc13bf03b0b3d630525cf824810a7dca
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.547,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.00908980426158007
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.552,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.00908072059340538
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..cd649af2316b8fd00b6dcab02eac48b3ec1e4402
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.591,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008977742846741013
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.5963333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008959169522662578
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0035016913a2bdee7d92bb4024b6c539fc08ef13
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.5976666666666667,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008954354670397114
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.608,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008914697075129006
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..1e2ed6436dcb93b9454e0bc9f28e9c3c9ce85d4d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.6023333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008936959925716909
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.6103333333333333,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008905164372580985
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..766373b71bf24a43c3ab707e07beb3f45fb3e422
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_boolq_yes_no_question_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc": 0.6023333333333334,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_stderr": 0.008936959925716905
+    },
+    {
+      "task_name": "boolq",
+      "prompt_name": "yes_no_question",
+      "acc_norm": 0.607,
+      "dataset_path": "super_glue",
+      "dataset_name": "boolq",
+      "subset": null,
+      "acc_norm_stderr": 0.008918717088507564
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..1cb0b214d8b865f3c374a470adc1aa19bfa84843
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_0.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.19642857142857142,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.05357142857142859
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.14285714285714288,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..380e141f2708bd7c5062bd469d54b594c8df89e5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_1.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.44642857142857145,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942398
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.3171545968156138,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..0f68fbe08496137c9cad00ba76ba2bb568875b13
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_2.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.42857142857142855,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06672848092813058
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.271744595274007,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..c71d62f7d7a0843d520b561199a473528f374e92
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_3.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0663363415035954
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.24603174603174602,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..17c771da8f019adac5a26ce62f0c9fa05e470027
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_4.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.42857142857142855,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06672848092813058
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.261867501304121,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..de2e147f2f30aa47439ebf575b1b1ead881fcde6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_GPT-3-style_5.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "GPT-3 style",
+      "f1": 0.235006435006435,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..fcd9ae9b201f07b1c6334c53f8687a2559913994
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_0.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06633634150359538
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.1940928270042194,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..050bdf5c7389e10db7f21e9a43249ce0d006bc46
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_1.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.2842025699168556,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b951ec4a268c6676dbe853103b7b4be59091f402
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_2.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.27010148321623734,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..59dad31055c6d853ae0cd7a6fc26915b367e4029
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_3.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.42857142857142855,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06672848092813058
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.3048219682039154,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8d3c039ccdf1a1e4a0c5a20c55e073416b181eba
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_4.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06741998624632421
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.3325281803542673,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d6ab678354213432c0c8d2a45c2c7e41a76c1948
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_MNLI-crowdsource_5.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.4642857142857143,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0672477765493766
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "MNLI crowdsource",
+      "f1": 0.28451178451178455,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9f44ad70dc998717a82144cbe947e88c2647282c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_0.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.5535714285714286,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942397
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.3737373737373737,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..37a993bc3b769b92452690fa9be944c62f1be747
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_1.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06633634150359541
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.29363017934446506,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6d285e9fbbbe26318c656f4cbe7ab5889b3a8e99
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_2.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.44642857142857145,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942398
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.2956393200295639,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..1a8da206d021928282e029b74885a9dc6256a3ea
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_3.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0663363415035954
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.25302445302445303,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0dfe03e805dbdeebccbd1dfe66199a226449b07f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_4.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.2593406593406593,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..47b325453335dcd38ffb3e9ac479f50dfda34725
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_can-we-infer_5.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "acc": 0.42857142857142855,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06672848092813058
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "can we infer",
+      "f1": 0.2854700854700854,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2fbf82fb39965889fe268dc0d6302e63c6e56aba
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_0.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.23214285714285715,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.056929390240001085
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.18837606837606838,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..96eba27102c7b4cfc72ae84af3de6b2e6af0a9a6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_1.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.2842025699168556,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec6285bca38bb557d37354b29ceaf0f3d504946f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_2.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.2807909604519774,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..61841c174fa83ed4bdc38228207d41836110f805
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_3.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.25852272727272724,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..17f17298c04ed7466bf18a23ceaf6cc9654d8f26
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_4.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.39285714285714285,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0658538889806635
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.2719797596457938,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d1454e81d55d16f4bb4df9b993f4d0c3e0e5f88
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_guaranteed-possible-impossible_5.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06633634150359542
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "guaranteed/possible/impossible",
+      "f1": 0.26059456040480705,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..aa2c7fbbca43967147de57e0912dee8b25b70b40
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_0.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.5357142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06724777654937658
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.3671394799054373,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..364b55e68066bf7387d272d3e36e2c4f78c70085
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_1.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.44642857142857145,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942398
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.3172825681224338,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..1609fac0cfeb5a811e67b3e8902ce7ce2b32be1c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_2.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.44642857142857145,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942398
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.2956393200295639,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..9b807c7a3a788dda8a11b756e0a20f363ef4f943
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_3.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.4107142857142857,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.0663363415035954
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.25302445302445303,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..cd659d6d33c44bdac6810365a1f27c968b850d43
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_4.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.42857142857142855,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06672848092813058
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.27129417747266715,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..0f661f597bae5c3828123774a22a8a2435cdf127
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_cb_justified-in-saying_5.json
@@ -0,0 +1,33 @@
+{
+  "results": [
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "acc": 0.44642857142857145,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null,
+      "acc_stderr": 0.06703189227942398
+    },
+    {
+      "task_name": "cb",
+      "prompt_name": "justified in saying",
+      "f1": 0.28883861236802416,
+      "dataset_path": "super_glue",
+      "dataset_name": "cb",
+      "subset": null
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0c23574e0ecbd8a95edc0d5b12671fbd13ec6be3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.6,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049236596391733084
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.04988876515698589
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7bd6baf7f7431bc86be653e05b43da5e15000a65
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.050251890762960605
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.55,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.049999999999999996
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0be7ca4e5cca2f157bd433ebc4553bcace8450e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.52,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.050211673156867795
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d2a2bf985be47e83f7d7f4001d373e4c2e48aa0c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.61,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04902071300001975
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..504d16587015224baec3a7b6ab50c7b0d156fff5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049604496374885836
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.55,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5985965f727166764bbcee6302a50d8a0943f2f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_best_option_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049604496374885836
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "best_option",
+      "acc_norm": 0.53,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050161355804659205
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9e4dd8e26660f1a1ea2df8e481e9f40875209e37
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049888765156985884
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.52,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050211673156867795
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..00ce89c57cba17628c4c5eda7c4730a3a11d73ce
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.57,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049756985195624284
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.47,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05016135580465919
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..898d98f892da427b7a40d58786eab2962c2ccc73
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04988876515698589
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.51,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..53933379c3e19dea963d49e08d4ab5a2096e1c79
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.59,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04943110704237102
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.51,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec2cbd147a13a4a88712fc0254174f04b15a1771
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04988876515698589
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.47,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05016135580465919
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..898cb7070f3333e94c5b9b48861e83711d2c10a3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_cause_effect_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc": 0.55,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.05
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "cause_effect",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..b62594cfc58fa1892213fa9a5342cc46c250325e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.59,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04943110704237102
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.49,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..917cd0cb8c17e0c0bba3da95f258d1e2e26b79bf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.57,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04975698519562428
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.49,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956911
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..32e416aac4c5eae0fdde6ec01e35a4b95a633ede
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049604496374885836
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.49,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956911
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..8213919a9bffd8c9eb157447c9875638b4e2bd8f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.61,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04902071300001975
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.54,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05009082659620333
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..58137f90901141f044d584734f9fb848a61f51a2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.59,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04943110704237102
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.53,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050161355804659205
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c50d0291410307f9443ad248133f0e67a19a0f4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_choose_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc": 0.57,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04975698519562427
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "choose",
+      "acc_norm": 0.51,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..658c716b408db5c5d04743e8b820e4f3b73a3d95
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.59,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04943110704237102
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.53,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050161355804659205
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf36fdfec13962d8256fa9d5b5be49ef6326791c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.55,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049999999999999996
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..4d4fae7d8c7c08224bca38a8c010b953f91b7a13
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04988876515698589
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.52,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050211673156867795
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d2c17f23b38517c0a80978ae4f94de34049b64c2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.57,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04975698519562428
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e8e3ecaa89ce16c36cf344203db25bb28c654812
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04988876515698589
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.49,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..4618fefdb8cd9fb12f5a724420d2361fb5decb5b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_i_am_hesitating_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc": 0.57,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04975698519562427
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "i_am_hesitating",
+      "acc_norm": 0.46,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05009082659620332
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0cba233d5310ca3f438365479bd8772974df9ccc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049604496374885836
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.49,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7204f9c7fa259e15ccf6a02a7d977373ccdd7c2a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.04988876515698589
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.51,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.05024183937956912
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5942f5906b9c3f8749a191aa41f47775237773a5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.58,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049604496374885836
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.48,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050211673156867795
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..9387bfd7cb86a5969c563f137658cf70ef4ca5b9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.6,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049236596391733084
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.5,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050251890762960605
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e86b6f1c1fd218018afa6a9cd4c1a1fc163b96fd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.55,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.05
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.48,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.050211673156867795
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..b139224de8a1474f72c9cd94eb5212dfe95cee2a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_copa_plausible_alternatives_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc": 0.56,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_stderr": 0.049888765156985884
+    },
+    {
+      "task_name": "copa",
+      "prompt_name": "plausible_alternatives",
+      "acc_norm": 0.45,
+      "dataset_path": "super_glue",
+      "dataset_name": "copa",
+      "subset": null,
+      "acc_norm_stderr": 0.04999999999999999
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e913e1a7a9344dab235ebc783f71d364c8b1a538
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 1.5651118840944627,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.042981320396581084
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.18521451898205324,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002457080525787092
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.27275459055962076,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.00326239965656717
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.21383961074239713,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0026815356045279183
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.06742918538311436,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0012528926958132866
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.0951561319414526,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0018453850908383695
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.07683031179945712,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014361637959827516
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.16070278930701914,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.00185948984697632
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.24114365421062722,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0026802570784209123
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.18671896323697262,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.002076974744205931
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.15871609393029643,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0020022541432733187
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.23173214485891466,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0026053790108179174
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.1825401908962132,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0021610595182852883
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6fadb4c39c0dcdfbf0bef702a2301d133c9a6f77
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 1.990645378245907,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.09426201816772427
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.19474596234384664,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0036808504078335246
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.17762533937937974,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0033135503178414513
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.17658380722184017,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0031889753436656837
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.05236928138258151,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0015077160489151036
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.04723848634979288,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013288114505525092
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.04696802025613863,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0012839248316351875
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.1402837628671216,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002748825869859068
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.12751328685413893,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.002450859209100979
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.12661225264501008,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0023435534892212776
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.16127567452387717,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.003143964896409528
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.14650506662968157,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0027996830850509635
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.14576477572483743,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002699050142839919
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3a970b9a0ebf3b8c125556470f6a07549ef02dd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 3.8652987845777536,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.10450428506636379
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.2928603436655027,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0033407712098847088
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.26267840583267554,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002974574226301873
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.2629920038581495,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002789298885039384
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.08172049965690244,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.001717030331861244
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.07276650009557936,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014807864762006147
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.07274573138257315,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014344798818270634
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.21347346743709447,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0026042923393898563
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.1899975461270108,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0022474664670417152
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.19024237456492324,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.002100380195663744
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.24332907877165075,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0029356367554022487
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.21726288370114252,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002572105790167164
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.2176826658743973,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0024213633519383406
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..86b3692e96caf4bff48031526f9cc6cf6586ebfa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 4.749488853686958,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1439737171039592
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.3295110755139736,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0030466187495971652
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.2908170186604444,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002713234606538018
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.29307632394991145,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0024677702395856805
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.09691059758420743,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0018267761852325394
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.08477572813016075,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015697017492672485
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.08511598673947676,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014976664233839329
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.24119075971078466,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0024282366700050326
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.21098836672574872,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.002067666204626933
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.21289346624936825,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0018914326319727048
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.274291969853511,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0027222499154415615
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.24075001746941485,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002369489900790716
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.24293590407668367,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002178523714675789
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..573d1eaad1eed9f2e6adbb747e9ffe7397b1d3f1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 5.176301731528647,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.14682428099423225
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.34370313232143146,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002947894794228507
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.3028195228554386,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002584602527686234
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.3042294531279061,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002297980823161359
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.10396227858967162,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0019360112695293232
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.09038662176400038,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0016265630560554562
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.0903654887337426,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015331774140601283
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.2524069371766252,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002404139964914726
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.2208918495013449,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0020294042523474267
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.2217428897230324,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017992319739905335
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.28720158613196195,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002676104542866642
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.2520155455904404,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0023002075133208935
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.25331485964949535,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0020731222812007586
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..00c118b96bd52a4bac7cfb04d4e28379721ebc88
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_coherent_text_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "bleu": 5.448542486611392,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1576603370541913
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_precision": 0.3534259465453393,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002910909198697379
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_recall": 0.31004812770308093,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0025222065017164704
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge1_fmeasure": 0.3118801144518462,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002229281342893514
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_precision": 0.11056115709540705,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0020146080400812697
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_recall": 0.09523241025254957,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0016332379953558076
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rouge2_fmeasure": 0.09565357595328416,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015669516035843776
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_precision": 0.2614140651257123,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002409323599550188
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_recall": 0.22758795096382134,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.00197916645130277
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeL_fmeasure": 0.22884790375880742,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017544092909052675
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_precision": 0.29587798141349847,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026666088968318393
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_recall": 0.2582635886545619,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0022422350235981096
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "coherent_text",
+      "rougeLsum_fmeasure": 0.26001996395899674,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002021020490775709
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..878702f29c5ca67ff0141e95f6c1dafd0bcfc2dd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 2.604286337933278,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.03591674116891077
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.2680956372841705,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0022592902002292296
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.370826122128093,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0035950144885036634
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.30133223661624636,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0024851056944864957
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.11163002440463841,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0011657527940780504
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.15654465364549336,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0018850260109515933
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.12592500811764892,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013089159002261316
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.20852815533900124,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0016092176061611565
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.2840330760519545,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0024031245269283963
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.2329565620227209,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001687724446352596
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.2355692548868321,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.00191083615835956
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.32127109053431796,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0028423630075748696
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.2633236447094561,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0020214976908264665
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..39f2a6ffe1d8107d8c54126cea3c6e68fb6d183a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 2.376958494986839,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.09315943625573786
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.2142240439442094,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.003669585143669259
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.19274042573599687,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0032419711801001993
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.19177598419259367,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0031175904013624814
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.05683805777177476,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0015258163112187955
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.05073039697609069,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013354368045998223
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.05060323801799519,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0012959436966401268
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.1553382299379965,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002742888570452596
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.13961389757181983,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0024018984589737314
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.1385854882700382,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0022957592520056066
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.17863698031283276,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.003158636767078994
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.15979139572794768,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0027474819161167415
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.15921566238118462,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002650725044809855
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b96e60875adaa91db7ec95776c231e021e491397
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 4.029805304404254,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.09060124588998114
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.3146026140303434,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.003123666505855817
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.27489622295236277,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0027426307774660475
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.27787798890903986,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0025362776452563375
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.08627383555029068,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017285779836978593
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.07473352232906243,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001444402605968691
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.07551724406985702,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001406834249268174
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.23138204856779457,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0025050345781738757
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.1997296995843297,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.002078456692172228
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.20227021958830982,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0019281583412694655
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.26264908396487563,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002801395123048888
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.22749287503171903,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002372211809509306
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.23048345042133192,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0022174670681384323
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..090efcff137c632a7311682089e0cb5633c9fb38
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 4.498804726055727,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.10775002870645801
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.3356969876826268,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0028462582702245168
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.2941317708277562,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0025456085630560714
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.2968322137022183,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0022769902890191806
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.0935059203110401,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017602062083097658
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.0818949185573383,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014958409085964503
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.08225064700727518,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014348926547678262
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.24614076393148132,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002283618951318419
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.21389679836058903,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019512939427380196
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.21588071480185272,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017373391626468349
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.27936714820247016,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0025524294154527605
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.24370834239777953,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002246504322912202
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.24604509499852464,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002021644244552477
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b54cd04847d22dd8250c4447e007757bd7567a3d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 4.927139553620482,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.12665338005793328
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.34744584188483457,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0027671133325864244
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.30598680841317655,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0024605634807434787
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.30772601506425196,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0021433656889168397
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.10081802718812478,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017973283857860576
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.08800270112485714,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015444421661352863
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.08832498480105934,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014651207993424452
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.2559711012561957,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0022826839200581384
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.2237171867711202,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.001933235277196951
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.2249364821771828,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016844326994501068
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.2906537194445758,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0025152309187473194
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.2548745214297313,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021861203156797256
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.2564962774637448,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019326551590934662
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..6245dd4473bb1c7a231e6feb15918a0b8436b6be
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_create_text_for_me_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "bleu": 5.319483620066844,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1937918156309555
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_precision": 0.3560891632971412,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0028052847609035393
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_recall": 0.3118566460746229,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0024563894577858706
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge1_fmeasure": 0.3137620150662928,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0021267449025118832
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_precision": 0.10787055205478631,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.001961259785243488
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_recall": 0.09277219175388018,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001599521459285887
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rouge2_fmeasure": 0.09313015798234085,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00152028521858163
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_precision": 0.2639125125532517,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0023507470257026764
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_recall": 0.22918005986102416,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.001931758070692363
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeL_fmeasure": 0.23055494373648347,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001676008041882916
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_precision": 0.2995336579782984,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002598520129287253
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_recall": 0.2608326939117505,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.00218886717995965
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "create_text_for_me",
+      "rougeLsum_fmeasure": 0.2627254077501295,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019364234716888252
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..16549f0a5be734b8fdb414188be828cbaba669ff
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 0.5554048543804149,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.04843400111692929
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.07490676533270745,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002150230516202816
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.06832821832134156,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022347623309728333
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.06971399215838711,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002110647717043094
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.04122768230311284,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0012955576500527062
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.03832509183940306,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013668800475287908
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.03866825902601929,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001277939861242296
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.07153482505420374,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0019220236379015598
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.06510051970164941,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.002019905690872576
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.06647105884856924,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0018913981605438715
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.07082417678060954,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002048106631259025
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.06475551139934367,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021371018812695345
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.06598366934280134,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002014393055687619
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..2b38c1b21611d56e2d9da9ebea91e9616ffc60e3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 3.1303501122849213,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.061431179427438594
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.18795241249491457,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019672359252996805
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.1822785883610696,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0021558750964836095
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.17861417240710098,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001841118798739779
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.09099657414603357,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0012717971723587007
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.08699074354785269,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013450179899541406
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.08617713780376496,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001215937184069791
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.1695073282851938,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014742554265132549
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.16297998873383596,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0016527556204933523
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.16060461647433702,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0013748351598955968
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.17092043784842698,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0017210240122471166
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.16483164473288184,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001869983093761878
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.1620576843620918,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016013095501406327
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..9631a8bcd54a0aa3ea208951fce1604b1bb7e3d0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 3.3001775192249134,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.10184239170439335
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.2134701829367398,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0024468758692696006
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.20530621471645766,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002460921154929664
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.20055771804060046,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0021701685544756343
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.08715705739965049,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0014291170688220639
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.08282189480619162,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014083407755700517
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.08158846564278203,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0012860200593204027
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.18060729349255075,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0017814219442781176
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.17257800926899938,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018037731355666897
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.16927544675510284,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015459458010864233
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.18766246584705398,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.00211000963150373
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.1797767907757674,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002115732933379051
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.1759439032072785,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018563724554560355
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..dcbdad72d2cd8f4c9e683a7e3060c794318276e9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 3.666475893279001,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.10987537074629235
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.23836199838819472,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0028469068712546316
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.22275349279619636,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002634091620880004
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.21946650594900366,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0024037209981410657
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.08677164701585942,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.001628328073416126
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.08034781449611338,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001485529414823335
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.07939963217690697,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013776332563822653
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.1931177821473266,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020670381912188582
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.18001529185227896,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019058042973943167
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.1775636766730472,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016879677128570562
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.20467508975739554,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0024391669746580877
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.19090226151707998,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002261425815356093
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.1882137601760175,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0020477882273726594
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..8b63e282853b7fc744b21d77a0afd7945035375c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 3.8809334302287417,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.13581356578783566
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.26018033627096865,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0029849403702905605
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.23824263498663878,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0026488517402878512
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.23609371210606708,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002433429420325999
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.08611475286824664,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.001635778374326644
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.07858583110015212,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014801568276794114
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.07785610993313638,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013788846901543758
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.20412964784769164,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021807066507413034
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.18679664087085932,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019178081474592414
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.18500764514280799,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017083269070961805
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.22044880753033722,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002598387740639484
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.20134137612920308,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002286825517593027
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.19959413484150632,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002096279733479644
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..81b947167b35c1f5024d5a7603f5dec1193cb84b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_gramatically_correct_text_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "bleu": 4.017338372139203,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.11094350038266719
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_precision": 0.2695313480190455,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0030112072928501336
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_recall": 0.24782182532367983,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0027049982514322063
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge1_fmeasure": 0.24439453610800832,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002471192570902368
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_precision": 0.08550020741401276,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0016123330184419132
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_recall": 0.07891710085141755,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015002516695171015
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rouge2_fmeasure": 0.07759858672584617,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014022028282609771
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_precision": 0.20874372759839854,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.00219378411400417
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_recall": 0.19212524503414735,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019738605195842055
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeL_fmeasure": 0.1891847721343501,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017507687945178178
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_precision": 0.2269415936324815,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.00261469191061651
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_recall": 0.20793572411416514,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0023144943247615053
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_gramatically_correct_text",
+      "rougeLsum_fmeasure": 0.20519809770842534,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0021184619122727845
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f5cb8cdedbb959dd0f570c11a67cb4bf1e0fbb28
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 0.5379698042344059,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.023682890094100795
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.11184434244818633,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0020578473204419783
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.17494853797920373,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.003281205101820411
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.13249245505272914,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0023865964860820023
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.018039929895774978,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0007516851699163001
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.03139481036037028,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014237434012642758
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.02216822438561205,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009397403776169823
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.07937245443292806,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0013055330499913698
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.1269667545228582,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0022135782119190543
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.09481463004436187,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001541902659571312
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.10436515332917147,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0018952448699960476
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.16392541157802315,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003068122098698215
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.12378548376730361,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002209146731463173
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..da114367ffea01869c31ed02d64fd38033eba6e8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 3.711647887806667,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.12239209464334602
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.3347678963130461,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002538034125678728
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.27699070131591086,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022531459521912115
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.28795658431823945,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001986436700440943
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.08380014964523355,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0015843319722248113
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.06976453338241907,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013297329518253997
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.07230219657399567,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013203982837554223
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2439230596700563,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020005873112574108
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.20072421941893917,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017248535505826324
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.2088938861271432,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001534187541306792
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.27665907100653375,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023050480585854313
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.22794132690389687,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001983838704481844
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.2373352341595687,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017982865567598523
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..a1fc53da9962b359d6cfb13d2996e803dafca356
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 4.158981847733467,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.10805920566927497
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.3434326875335696,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0026992899622732525
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.2865272615258301,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002262834986590033
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.296355770492695,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002043539678118385
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.09254926113645746,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017008623486038437
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.0763624864031587,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013832865499576604
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.07905711793730946,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013774541250299033
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.25255081340500835,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002168002738840324
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.20966503552217744,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017709941856742383
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.216815482854756,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016012856900651986
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.2842764554431881,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0024381683573090783
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.23649066236480792,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002022905140524907
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.244683933643628,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018556926092760141
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..15662b9ad0840607da522a7ca142dde352863cfb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 4.7695767170826615,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.16549584151536006
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.35625735674057685,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002753457728280898
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.29265236837044695,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002280330795443891
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.3035075966249268,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002031043294823014
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.10577742526833883,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0018673396392927667
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.08549991950256323,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014672056887335024
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.08864387470534706,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001446455854434278
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2671981244809395,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002307047675439304
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.21813281187586306,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018116496367021734
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.2260702121301555,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016207565003713668
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.29969790030371246,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002551276392318688
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.24481991830895322,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0020451634619149594
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.2541669303215731,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018585514707850925
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6584d669847862a22902d7ffaf87d468ca69867
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 5.0994041530025545,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.15184509857797265
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.3625581772528726,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002820926829290532
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.29556687359417316,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023030954598190034
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.3069669703069319,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002055596225803597
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.11265300538187106,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.001984841845142737
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.0904636966530554,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001544958177720571
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.09381098739864982,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015224351850222366
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2731899620101324,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.00239084885412292
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.22137993241122347,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018642397142735888
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.22975500104331817,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016770985431782879
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.30531248482102935,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026231141349939897
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.24781445089520615,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002101182002409543
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.25753199178665565,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019112760465508692
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..138cd92c265be6f24e2bf6d517f28805b57747ee
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_generate_text_restaurant_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 5.251573482634308,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1553666643851091
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.36700582996097203,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002866361764312095
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.29908671329451914,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022842305263675784
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.31100455847956715,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002052592278425154
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.11715256391592369,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.002004164016012016
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.09355545687055947,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015451237546724788
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.09730358308616667,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001525935012941263
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2792795885938201,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002414534192644447
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.22654007734086326,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018601465471659855
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.2352915422950905,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001672302120038204
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.3101815290461886,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026513885447339244
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.25160776782553634,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0020719183900535583
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.26183889011003636,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001895806329444741
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0b5f1973c52b8c5cdb302f1a90e12ead26221d71
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 3.416746947803346,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.04915567383423248
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.41219549454005344,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019884907431696104
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.49154498554014486,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0018555667598375688
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.43965168599123006,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0015705219676545596
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.17378317584242825,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010901893538702665
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.21285050587435922,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001428926493666674
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.18707996026648183,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010713647179164203
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.3073557477422296,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0011789635478387738
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.37811034759792467,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019549694924619787
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.33221821033645077,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0011795568835039534
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.33947329485158056,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015891394303630454
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.40971450190173514,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0018376040861944267
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.3638009261252252,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013481348026921308
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..921a412d28fdf7cec1f08ad284a4e1ed38c50932
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 3.8370970182392985,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1144591225447721
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.33744773531264344,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002678432208002564
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.2779081994219456,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002272364632457066
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.2885492968351212,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020292469178824456
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.08918695736438806,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017352721360830848
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.07226568712760938,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0013494607128344402
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.0752688547278534,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013552863861086071
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.24853209420474298,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002175861850486939
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.20284289627233723,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017347865605118378
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.21099118017935053,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015717685572481305
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.2808597038267283,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0024612171563718217
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.2298552776553825,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0019962013821171482
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.23909437838411668,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018320148819675503
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d5b483cd063fb593edb5af8626e44bb70e0a3ec6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 4.373718657948969,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.11887879947848155
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.3522552603743486,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002763858033996764
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.2889714759845472,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.00228492884684045
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.29973019758970454,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020502121064942818
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.09934840296950617,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0018189448808929704
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.07999739923020184,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014210909684714815
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.0831703337253987,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014123449794312788
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.26226475143890166,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002301455268301791
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.21294988420936256,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017815097701725465
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.22110973690939364,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016159023437830125
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.2946849915708113,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0025524376570338707
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.24032554668210832,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002050233303327457
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.24955436884636814,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018809736592006266
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3f747dae9037afa78aab7c7cddf5d3790d73688f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 4.872872536054477,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.17994394529095062
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.360735661960322,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002835390448997847
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.2944152190187286,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023207814101633794
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.3055513704733046,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020673466657850498
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.10920157678214337,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0019434279841216329
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.0870860469506747,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014872544127445687
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.09057857385566902,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014742462627991503
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.27244358337055397,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0024054386143698534
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.22030127184406081,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018506185371882098
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.2287609415460662,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016617607367168975
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.30429745199273023,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002612593595236828
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.2470915077136621,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002085484889135651
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.25668504598663233,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018889983272758794
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a57e606eb8c4ef2be8f6eb9dbff2fb4280000fed
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 5.212454694507268,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.1288163730607949
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.36456574097223493,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002845913293926449
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.2989078546481851,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023201439477763606
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.3090786443943064,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020670761005209533
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.11386196869648488,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0020442207769660018
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.09115599764568338,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015501355013158944
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.0944527466973476,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001527049255606511
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.2769682674664966,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002470609873235701
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.22450967972348715,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018729810994751136
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.2323973425908195,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016905329597190666
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.3089928251978803,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002660795958922366
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.2515644789803418,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002098522007442167
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.26051491605603,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019072972536883094
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c545906ca85878f54f70131a81e395319e3b87b1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_e2e_nlg_cleaned_text_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "bleu": 5.326249606769673,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.13870766757762862
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_precision": 0.3689871087097193,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0029017082283349553
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_recall": 0.3001344146565885,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022905081784066715
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge1_fmeasure": 0.31118667832544594,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020471931429837612
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_precision": 0.12022586215655875,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0020979416142716933
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_recall": 0.09432946255117527,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0015316319038071217
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rouge2_fmeasure": 0.0981873842191505,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015253172592372794
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_precision": 0.2833630391152307,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0025014801176794616
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_recall": 0.2280006358827552,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018282950889621254
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeL_fmeasure": 0.23662330266166123,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001654720258750121
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_precision": 0.31356761141424766,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002691111456370736
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_recall": 0.25378328346517437,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002063376374536576
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "text",
+      "rougeLsum_fmeasure": 0.2632319188900435,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001875582937147226
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7a545e1dd3fcb5f529378ddb69542946de913ca
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.07849100625603515,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0013763236409170894
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.18830692834903057,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003201584532139998
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.10930956321205572,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0018636365300419618
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.0101373697747153,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0005725799485965975
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.02554990695060758,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014536715782248486
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.014331752023111665,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008038622113235613
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.07153361180580091,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001183235553725598
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.171439117832265,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0027508968911593348
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.09955783945841405,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0015927736901503137
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.06237679092272722,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0010796552208218652
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.1514930768097993,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0026421209389933848
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.08714871620375533,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0014790422117723686
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.47457826897610184,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.06622527858136913
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..4f17be8a966b2b6a5b4e13b67bae9b884b2e3678
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.09609231578553688,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0015668691591351292
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.23552842368238433,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003577814966154032
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.13474307731979848,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002111986178677571
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.0110354532279438,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006463903956640263
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.027990471006616467,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.001658813041921148
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.01563559121007456,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009139886175241735
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.0734302137042163,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001138075001700217
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.18134020183601798,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0026912992930091363
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.10311323732086543,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0015375375103156618
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.07630876213778368,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012122651784603749
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.18889813705308528,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0029100234368416634
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.10726294947450196,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016523566097021156
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.6132540544643678,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.0713885076138697
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d7a3fd6fd24ba90fd4127b0c36df67ae8e2ba85
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.09383796110256946,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0014865897795411922
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.2300444944691374,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003392043854938688
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.1313944002337063,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.001990658173774306
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.01098830723574462,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006303776417170931
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.027489027406269212,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0015570004325674606
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.01551951461927907,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008825861261678188
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.07370182535719132,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001100110897417761
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.1817594002432442,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002569209339964884
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.10329061631237338,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0014671878058074702
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.07447014880573084,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0011974801492887503
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.18408290764630678,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002832630645866158
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.10446611375407802,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016139685037892917
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.6167432807079934,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.07313238207352012
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..36f07cee8f0916ec463b4ca27c2e88eef5e61d90
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.08929232207008256,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0017661136394464746
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.2083714353056586,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0037507822542893463
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.12223241280960181,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0022597737898330176
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.010779714787610186,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007228389047199584
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.024684029325778328,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0015436289252720226
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.014615300016460326,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009436727425446628
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.07198363558556994,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013768675700732372
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.168224053314788,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002856247526588458
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.09843060072407699,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017199014486665768
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.07122439300395828,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014339663669546733
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.16714051967493118,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.00304885339797809
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.097565135698147,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001816709165840859
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.6537439527112324,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.08789003420216455
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..76374fde7534d9c2d8a953bc1590dbdd200279f0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.027438753007734348,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0020209855248670377
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.047135286894855855,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003093201466426986
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.030623951982172243,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0019723679586947157
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.0036391643178718376,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.000651422116785187
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.007018854013603984,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.00101986519717702
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.004331423800367673,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0006498030956306457
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.022758262766782358,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0016467261403303932
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.03973720943624368,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002596893758975351
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.02559588917107183,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016311116927956652
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.02236476517938642,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.001641725788842373
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.03877314043404623,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0025919403907963476
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.025026908052136167,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001623239492581538
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.38568323827085005,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.13140258824575432
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..23f4c1ebcd51ac37d12fd6c68b545c32303d2ddc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_boils_down_to_simple_idea_that_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_precision": 0.004288164665523156,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0019144304746682786
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_recall": 0.00019598446840121887,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 8.942783707027923e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge1_fmeasure": 0.0003741452934927028,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.00017038377957226348
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_precision": 0.0,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_recall": 0.0,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rouge2_fmeasure": 0.0,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_precision": 0.004288164665523156,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0019144304746682786
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_recall": 0.00019598446840121887,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 8.942783707027923e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeL_fmeasure": 0.0003741452934927028,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.00017038377957226348
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_precision": 0.004288164665523156,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0019144304746682786
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_recall": 0.00019598446840121887,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 8.942783707027923e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "rougeLsum_fmeasure": 0.0003741452934927028,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.00017038377957226348
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_boils_down_to_simple_idea_that",
+      "bleu": 0.0,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.0
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2d1076b83fdb82ec9af4da055df2aa742eb39e64
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.07695974919266163,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0014773165827520049
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.18662101528755234,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0034322790231989847
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.10741115659863423,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0020029906467731195
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.01122477293725975,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006155567811512096
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.028017590709361054,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0015078040578957851
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.015793527267809353,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008540265285033742
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.06992993909202184,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012721116443108543
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.16999666228834834,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029761829036210304
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.09765196595353162,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017221112550801897
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.061013847977830764,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0011300630372763191
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.14954259216443205,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0027335378592026204
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.08537020359960786,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015426696504085384
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 0.5835098927919238,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.07719313039217032
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c5ceb0dc0da1efc58e9511d62d6efea860436a8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.0998197720955355,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.001791958979970089
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.19150976792164776,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003597233562430454
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.12292814399550107,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002063269026074902
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.00926872748126837,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006094283518785105
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.020793570369667533,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014099011327662694
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.012260258344477375,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008029144977571671
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.08018556273037858,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013905565313662618
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.1534507156318649,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002735628237761709
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.09847662810007793,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0015558437843189248
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.08017202493423412,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014158245846160884
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.1538977801609464,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.00287738921463875
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.09851599027450329,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001607691573132454
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 0.5393663863983892,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.05348433562375772
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7cf831a3ddafdffb35c065e661e660afa2d747e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.09588515299897513,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0018389413668098858
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.1941351180627413,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0032321812427151307
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.12080327979692661,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0019325844096013765
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.00963799324260621,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006615867592044398
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.020245217917730207,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0011797301011867512
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.012201390939302973,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0007193995074256504
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.08057175111690566,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0015714221610920258
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.1632996891477624,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0026283320276119515
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.10138824592737848,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0015778531626384088
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.07734192765900497,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0015709846641160572
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.1560636734165814,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002615177895498355
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.09694968298249196,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015608436285872904
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 0.4248411619556655,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.04337063842371632
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3cfa33afdae110a1bf7e2a03dba3d5e55a6719f8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.0979524500570362,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0023574213995995338
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.18432267940454714,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003449394808563609
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.11838286391586506,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0022575984998313976
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.01223431053167332,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0009139707393864935
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.023333668667089247,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.001381718229050345
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.014849672103797189,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009398723971147706
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.08382542142444173,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001948560716573151
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.15921404653049828,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002868932069201499
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.1015835231933467,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0018425036048339849
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.0796692729044991,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.001955663224730415
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.14966156017894405,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0028188651661202858
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.09581512027677436,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0018225664745385676
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 0.5384672086268061,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.05628261735256145
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..424add2ddeb9911027cc2933de3b952650388fea
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.03078887567599419,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.002267198013309019
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.04728638198757095,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0029563776305562995
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.03268274482045124,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0020159690172151102
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.004097645217219916,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006352569090777523
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.0070103107833449875,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0010527888125515437
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.004600261742688949,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0006815109118145586
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.02580787661904382,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0017950648224843338
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.04086841692598574,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0025615977979740897
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.027882226495475622,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017012154601545335
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.025250388596127913,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0017916684649173303
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.0392238241311461,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0024646800606543846
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.02702176662269746,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016712710261589593
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 0.30242530199415063,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.13172883284690357
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..21ec87c5929f27370bae8de3eba4ca0a35ea2015
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_DOC_tldr_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_precision": 0.0024946317337081986,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0007459407337053418
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_recall": 0.0020432834635293762,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0006122723724231832
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge1_fmeasure": 0.0021779269831245112,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.000636647151495716
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_precision": 0.00015842386125404991,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 9.164096613628691e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_recall": 0.0001078167115902965,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 6.282744363562356e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rouge2_fmeasure": 0.0001270035372229107,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 7.339490340552063e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_precision": 0.002177974195907487,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0006242303095573485
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_recall": 0.0017464607065091375,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.000498369445585121
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeL_fmeasure": 0.001888812253805427,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0005328781795731039
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_precision": 0.002231576254226526,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0006354831460676118
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_recall": 0.0018536648231472165,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0005527691709280919
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "rougeLsum_fmeasure": 0.0019602816648974794,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0005561333735819992
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "DOC_tldr",
+      "bleu": 7.569559051730558e-44,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 2.3751737130499017e-36
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..b9a0f5cdb8b8f96060621644acdb30d264f45f98
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.08684854788612234,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0016136017598306646
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.21068678976862762,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0037297139710485957
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.12137792760871402,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002184184971255157
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.012636632651205262,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006618849333669708
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.0316888941525413,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0016796888756788677
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.01783873426577586,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009275277075872498
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07624380858300416,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013077167367173933
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.18587184963214815,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0030939829912028922
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.10669020751320313,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017737444033652681
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.06773047218424798,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012398806155871941
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.16635584373488718,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0030019144404050606
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.09494249315705383,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016928172404188113
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.6141824165767692,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.07244482599281088
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..88a320f1f3c82d9ab9b6ad40de55f46b428417c5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.09856438801955326,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0017913961749217754
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.23807395731457542,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.00408145979752075
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.13729478048942775,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0024098935839123887
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.014805002932563531,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0008137316866053345
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.03704847337143438,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0019513225466723323
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.02086687296490721,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0011212733091223604
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07886562535884908,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001319440885298338
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.19199210330518746,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0030607720042053164
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.11005303698684504,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017730014084980385
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.07731479241226244,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014429529345158431
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.18826403944987077,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0033888132406159205
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.10790506830750389,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0019575871972268514
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.8152552954236955,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.1031507916283277
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e4e3b5fa48757f184660c2b3c8a126123580eae6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.08892601657513456,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0016077156412850602
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.21529785402215762,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0035920176085112838
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.12396034301655152,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0021408486311541627
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.011265761012393422,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006031851124179686
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.0276901962071342,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014341330838498428
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.015790474040565182,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.000831397853071339
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07425540521394834,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.00121183146821238
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.18108047541087044,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0027630960930183058
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.10371163259608823,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016126497685603005
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.06968809353220383,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012730861197946873
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.17037941440049567,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0029297652528551157
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.09738815767936584,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0017074564850241471
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.4783982759526829,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.041614834308327325
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..362628403fd4921aed24ec5e6e19d82bad853229
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.08435225307357676,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0017949929630033349
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.19524716118009153,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003729643037639876
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.11451534571985687,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002245243404997308
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.010931147147210569,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.000631905230395927
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.026021044054339443,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014337475560734076
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.015065187296172565,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008474739343157203
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07171273980778682,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0014130706113843967
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.167240338703402,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029869789558385
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.09755266669239072,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017557981664671374
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.06667665265151085,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014209048598926293
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.15545938043826846,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002970962161003125
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.09051613592037677,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0017384555593769912
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.5544562884765827,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.03868981500279069
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..0cfa8e273f736f5ae8d6f231792f6529d47078b5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.028606404294185678,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0020371981833308224
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.05177166241618001,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003118570625917115
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.033226658072660585,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002003945138254869
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.004053086557676976,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.000644383420856288
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.007133537400455768,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0007785336270850277
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.004530040031767069,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0005161711108430996
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.02472463147760117,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0017881996232063642
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.04459955593168214,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0026285803260540815
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.028498538822695166,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016689936239707673
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.023409670499661615,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0017830228948821282
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.041408985247042875,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0024949239488166234
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.026611692302705815,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016132169063633166
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.23195231943321903,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.054461911286189185
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee329742299ada674e6d871792602277a0cac59e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_article_DOC_summary_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.002544888964583456,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0006804984372875271
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.002139575841226647,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0005562502135583049
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.002279670952470671,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0005995355838360033
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.000239072186907925,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00014515151101602653
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.00019053273770254902,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.00011332476106214262
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.00021112542779058987,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.00012666146238130252
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.002161847481010399,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0005749806010887603
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.0018323415576938233,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0004753823971115864
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.0019409469526661608,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0005073885489964555
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.0022997709059248868,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0006186158865403682
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.0019535380516075003,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0005119627013983172
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.0020687632617786135,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0005471176517107834
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 2.4598586849321513e-37,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 4.3357176079980746e-32
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..f9711bdeeb20e4c5e57cc60b96a209bb214b0fe0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.08294449475355638,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.001501101677358941
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.20318981866441785,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003451770742452455
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.11615011773689751,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0020132702860050805
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.0115654770629999,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.000620636009223327
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.029012948159446418,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0015609513028227002
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.01634264454041957,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008732318540767917
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.07485979068030396,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012592589840183614
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.18415577121037147,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029429725468367554
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.10497029887714841,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016965946158159747
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.0654649026019978,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0011528108608884256
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.16188192696498585,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0027459266075334754
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.091932294749694,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015639138169942146
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 0.5173175787874577,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.055369098931755845
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..6bcb45a2ae542fd06bfb5c08c488144349eb6a0a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.08748874292219283,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0016046785625036608
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.21040419098884006,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003596032405414458
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.12132677528862063,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0021172474639601026
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.012177718487699685,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006600449164619634
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.03041617462460392,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0016279180469739754
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.017118068362416517,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.00091598354274057
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.07463066246768363,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012825612167355926
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.18070029655502712,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029520412330890744
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.10363956196342357,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016890572561683093
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.06928118345477047,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012737323592854019
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.16792413773506376,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0029340056636205994
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.0961985987954344,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016792470214151791
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 0.617354720838304,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.06307890231427365
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..019431ed0dfc426e5de01465990edcc4f9fce773
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.08502500988635123,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.001498937813320831
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.2084492768665317,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0033474975083069305
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.11881862682601178,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0019599355384508203
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.010957922059575794,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006364547528005359
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.02745994543864676,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014679599804070377
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.015337369323206902,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008247870081098087
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.0739732762619997,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012237819610442023
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.18227843033716665,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0027816909692955204
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.10352955674946399,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016013916021151678
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.06740414152435432,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0011978138296961046
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.16720075001064724,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002795799505244066
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.09449014264304802,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001581094844660653
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 0.528150461312332,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.04887622861962553
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..585f05758013b0c6735e2ab556aac6b151eed64a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.08286382508828975,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.00171211565369596
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.19417127366399595,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0035347086651873116
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.11303815097050755,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002085394908569434
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.010709281907276956,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006463393919380488
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.02563055239095909,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0013753043663831622
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.014667451968252554,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008103234054454378
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.0723252465078549,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0014008899550456907
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.17025301778061233,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002944125742781433
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.09884209508770671,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017207139349791946
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.0650622060288472,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0013556680905752983
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.1537510648011344,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0028634633132608622
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.0888642550425071,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016417562882641062
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 0.4932998587527608,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.04917973267537893
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f58fb1a1d65d176efa76889071cd3c5366c0739b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.0299789202555777,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0023310463175663133
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.05033299021370721,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0030688602223686704
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.032765818131531135,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0019956975528504003
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.003955464420202306,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0005645114147565559
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.007608591554539119,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0008849021378583971
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.004765891573358166,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0005748770709346177
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.02585211610210547,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0019823318353263796
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.04414951473810684,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0026739955507568457
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.028427633970046932,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.001695433517325568
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.024288617878882954,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0019683052842310715
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.04024052293200098,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0024657230773210245
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.026135699640989368,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015948963052667224
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 0.3152874467334013,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.07390349309399162
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..45fd0a4425400f18c76051ee3be73534a05b04ef
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_DOC_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_precision": 0.003145834181525519,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0008641359171585344
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_recall": 0.0027491540556537565,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0007447118682849743
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge1_fmeasure": 0.002850102432403856,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0007630282877736268
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_precision": 0.0006187008822968868,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00029194397834022607
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_recall": 0.0004933724831937344,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.00023534159735572293
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rouge2_fmeasure": 0.000538565784940234,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0002527457661965069
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_precision": 0.0023242531040883747,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0006266425117623151
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_recall": 0.0020438981820361424,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0005399846248395976
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeL_fmeasure": 0.00211639976529527,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0005548761683122237
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_precision": 0.002621636659839826,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0007128110133984216
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_recall": 0.0023514083301448595,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0006536022122755625
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "rougeLsum_fmeasure": 0.0024026672428148917,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0006455426387014536
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_DOC",
+      "bleu": 8.230901975344253e-37,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 1.6818144591792003e-31
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..25df164293bd8700076b60fd15297aa3c79ef4d9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.08101405207575763,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0015402709854789834
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.1978371464700832,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0034969455098272624
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.11344123075110753,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002075752711472246
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.01181630825554818,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007223475673028841
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 0.029271962971207783,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0016894106447306921
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.016637993147190867,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009926063504469168
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.07385599916218046,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013240985175350677
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.1808620333503495,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0030065019141283108
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.10349719856532423,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.00178021572550937
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.06323384533522881,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012031428852023567
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.1564882734228425,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0028531783790494997
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.08884476245308559,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016345699043350426
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.6162431159385546,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.0719411252845223
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f82fa9616ca23fcac8ffe32d087a097b758bef34
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.09212077005868066,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0017630634952107368
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.21369522061211693,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003729436274385912
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.12537834249633983,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0022469691506953465
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.01206376214517874,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007516662252398531
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 0.02903754360252403,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0017031825569670093
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.01660477741264796,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0010056200276438794
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.07655663031331215,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013855875426255315
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.1792751829195347,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029860001198346964
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.10446236594583992,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017570999718229995
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.07232336786129806,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.00140309509381785
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.1689005605679794,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0030087684057466145
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.09853403043110821,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0017820950507240858
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.6233103019291443,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.10322680839865134
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..092cb09dc061ea713dfd738a36a72bf9bc5acc43
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.08661269663304301,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.001593838459943478
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.20717931900250192,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0033843575265547453
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.11974042458990046,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002030784483754028
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.011007610065072464,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0005989306753914806
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 0.027125371681868536,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014187760277359451
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.015360819588816904,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.000809551101724943
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.07395213433806418,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012575784598958418
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.17867696254495863,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002758667555218375
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.10256512747901998,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.001605784979744158
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.06701000937643628,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0012330207864759619
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.16211217586867194,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0026761095348479098
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.09286761848139825,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001559907403116052
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.4996183843525882,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.04001578639941148
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..a66f7ff77b03359794910129582a9101c97049f2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.0806895748479458,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0017148726001275283
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.18907123674289394,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0036401681772858063
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.11040951377527043,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002179873874618349
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.01053868364581689,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006210803937322794
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 0.025668261986885543,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0014569341217374899
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.014649332777625616,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0008477225467246834
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.07010786798662083,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0013932486949560677
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.16531909956669813,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0030227929410000416
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.09609527909787907,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.001772609518829833
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.06390479614025571,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0013316010773909046
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.15100544107920313,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0028997429382766494
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.08755892275631494,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0016919775746021333
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.5195966300461277,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.042459426003985785
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..007ea78c03caea076f26bfc58e09fb9438eb7dff
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.02658254011537247,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0020748261577405607
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.04377731709316155,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.002784901327388771
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.029279070120697838,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.001878769664661074
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.0044766254520966255,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0010428981904857266
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 0.005945068157009263,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0007338081846565158
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.004119906223301282,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0005578664016300015
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.022618399699937414,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0017607024402004165
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.03769542551971095,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002353427232761915
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.02506826654645877,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0015851273432252462
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.021501210074057614,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0017408075966930808
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.034957744191750476,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002217068454307888
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.023443260455568726,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015176085253178068
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.21210378195277477,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.05267300095484724
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7452cb58f819ea5dde181637f6b63ff15c0c4e8a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_gem_xsum_summarize_this_DOC_summary_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_precision": 0.005288736420811892,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0018191165622398336
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_recall": 0.0006712474238977358,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.000231168077080196
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge1_fmeasure": 0.0011581184102891899,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0003946303405826163
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_precision": 0.0008576329331046312,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0006061777330437759
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_recall": 7.576832303743922e-05,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 5.452796435684482e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rouge2_fmeasure": 0.00013885485583598793,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 9.966211058781627e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_precision": 0.005002858776443681,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0017037398901424319
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_recall": 0.0006283657772425043,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.00021043869960245262
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeL_fmeasure": 0.001083541633497483,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.00035783272085016286
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_precision": 0.005002858776443681,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0017037398901424319
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_recall": 0.0006283657772425043,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.00021043869960245262
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "rougeLsum_fmeasure": 0.001083541633497483,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.00035783272085016286
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "summarize_this_DOC_summary",
+      "bleu": 0.0,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.0
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..65cf6ac8481903ad3c0d5e36036a9f4c7016dc96
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 3.656755366303797,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.14837890695680994
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.05669470574943534,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0020170312039152695
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.5066296296346544,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.007659513774926616
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.09106715079841605,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002391471928167312
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.03860274611467539,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.00172983035557276
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.34854187631620626,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.00814423352083145
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.06256105007088962,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0022953455816733365
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.05563894642256185,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.001982960805433701
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.5011913179283217,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.007700971199463536
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.08960277978128786,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0023665012918680627
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.05312130935399373,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0019781535292318657
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.4801456290593825,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.007794216102275114
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.08521160683074443,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0023490297225075376
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..8e961e24227ea5a3d620ad6acec9b7df3ecd9a4d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 0.8830608958021205,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.08562321028264006
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.03607709470611491,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017441028278732172
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.1952470751009191,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.005487103316834906
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.040647935659745706,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0014550077414889832
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.007972127932130054,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008580787510736185
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.07120120024573734,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.004600441634483666
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.012117898396108417,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010508174756117949
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.033979273307145695,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0016175369455391048
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.19294098495627118,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.005472126990576816
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.03909307269627734,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0013947783444551163
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.03227030109817833,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016612299817095036
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.17402891178983584,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0053066479466079685
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.03550984850410223,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001363844016926205
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1491cb49b8ee78d1877c7e6683e2c18ac561fab
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 1.4935006686116292,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.05443694683302151
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.05267853270925152,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002449745789546971
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.24860214672787803,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.006500686399418739
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.06032529701781306,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0022263011442644493
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.016806908973702062,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0016553589028564524
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.11666497774928083,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.005908602168091971
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.023433042203883557,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0018005053734734208
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.04925134882919058,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0023033509345347773
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.24482036799051923,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.006496173955774258
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.057693738636468685,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0021627311467575755
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.04806464483998574,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023691482012047245
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.2255921129994038,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.006372394561834973
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.05423256001424035,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0021531740526494
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..5e0059227b456f5876db326bad995282bed8f147
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 1.8212244525264603,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.09849562818662225
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.06508938178602254,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002888335161689235
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.26659804850838303,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.006676058649232921
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.0723460691197474,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0026163497556041058
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.023241183787932676,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0021132241633176545
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.13124281702649712,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.006161317732310535
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.029903412643591754,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0021799636798326538
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.05994865949151067,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0026955181247782334
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.26144434776062075,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.006677302291405039
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.06828781879330677,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.002518729485584185
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.059049668114767744,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0027649257923785986
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.24459992364434763,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.006585924906880039
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.06528793058605904,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002529000694923454
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..f9596c1b36b8e7b162b1381eb63f59b3c80659bf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 2.0235091580656346,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.06992480281711495
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.0750764899008003,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.003250878714640169
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.28071950983923893,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.00668526364483302
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.082090911255068,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002803446342789559
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.029562846153838467,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.002443964528445172
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.14150750671815635,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0062328605351622945
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.03508516074414074,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0022612345217735254
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.06986898167803665,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.003066369740945722
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.2749317176165797,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.006684573146158844
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.07771533962795445,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.002693141545803596
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.0688802573137848,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0031419995471746284
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.2583791181007065,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.006608286981904223
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.07481316087741567,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0027185549354669615
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a10a1eb2677812c46b6a479ce19bd7afc9175fb2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_Correct-the-solution_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "bleu": 1.9681135945636459,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.13208724741948602
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_precision": 0.08049093429476127,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0035142461324920273
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_recall": 0.27883210803766567,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.006581683130296607
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge1_fmeasure": 0.08562740019962164,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0029691098655082715
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_precision": 0.0333369150257513,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0027845366791002944
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_recall": 0.13700194089143813,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.00613423127185248
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rouge2_fmeasure": 0.03697008601032397,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.002496780466003745
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_precision": 0.07459959278412087,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0033268926264176904
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_recall": 0.272528207623618,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.006590906751190863
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeL_fmeasure": 0.08076583460906837,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0028795378570412307
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_precision": 0.0735499511307004,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0033973926098363675
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_recall": 0.2546069662211679,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.006496163637745307
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "Correct the solution",
+      "rougeLsum_fmeasure": 0.07760507931200265,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.002889469047971454
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d98232766e27011bf91a39f35786b6146673952f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01166526473007815
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01166526473007815
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e0a4cf0184e3825a13dae0b208716594fbcab85
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5016322089227421,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665762007194868
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5016322089227421,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665762007194868
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..b47769a9d503c59db4564244ac5c39113ed92aeb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665264730078145
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665264730078145
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..6ca85959a97ab37a6d04898fb62ca62a1c07038c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5272034820457019,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011648545262429021
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5272034820457019,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011648545262429021
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..dff08c704bb0bdcfb662c07d011abf04563faa31
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5125136017410229,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011662170084916898
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5125136017410229,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011662170084916898
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e55fa42fcded289916b3d3fec5b7884ff502a35
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_choose-the-most-appropriate-solution_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc": 0.5130576713819369,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01166184537588635
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "choose the most appropriate solution",
+      "acc_norm": 0.5130576713819369,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01166184537588635
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e7fd8e6eaf468b94ad9f5c9f86d28de86fe052e1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_0.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.06614185025884788,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.006992904774416279
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.016184382076826428,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0008939401678502145
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.14173571994695022,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0030334268104372647
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.024878431671277717,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0007084095842894381
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.0014202808708619317,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.00013696486457806
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.014499650869656043,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0010689452794773893
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.0024039653739287383,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00019957865940879526
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.015195809153249718,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0008191061120771218
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.13579609932870865,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0029357166668511343
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.023587495220817104,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0006681752509678566
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.012906484148544855,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0006299772819920492
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.12046291801697726,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002609561210140041
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.020150738338231722,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0005489881127507307
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e64f4fa75e3fe858641e1ecb4edd02a1110ae086
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_1.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.10718681853330272,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.018000911234593346
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.04132674547577212,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0024485259308753586
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.08689740514715649,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002916092372055045
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.03703813399564072,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018091336796340791
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.0071242151063781395,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008842251520965283
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.011318163060566497,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001006748832876741
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.005941822792449927,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006087286220180097
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.03625394677542022,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021263148272352293
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.08136452827721419,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.00276242885759181
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.03303428861037457,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015956649904201666
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.03652614662967736,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0022174820770640377
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.07577016170577146,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0026023305739597473
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.03218887137036506,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016050372004655393
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..9335d8c00d0f2139efbe99ccd31dc4d064acf334
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_2.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.1712273909909647,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.029845744864029765
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.045080131535418684,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002665199214443293
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.051600544061536885,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002562420942904496
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.03482910285332883,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018619215722441118
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.006360924839859195,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0008787433792928841
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.007684278380676342,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.000961471039023744
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.005293218483445248,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006663810940536303
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.0394285879549089,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002356887213890825
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.046948342759142385,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0023931560766638333
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.030614463329020847,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016647185279519317
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.04033700739178538,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002458798837565943
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.04587514391567024,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002316207868651937
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.030839298564677823,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016870567056580634
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..9bbbda3b31fbd97eb2c861ad1a84008db90a49da
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_3.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.22910425239933835,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.023846299745637276
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.04471001776599436,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0026879796556863317
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.045685551243527515,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002440523184391023
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.034617382780722655,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001904100064118123
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.006631747554370845,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009077372437363107
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.00708411646538401,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0009178652859708077
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.005307127222312835,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006509675188338064
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.03921165897731945,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0023674286187342075
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.04101390938445623,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0022350756769902065
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.03053639049565289,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017070590062506794
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.039886138935695335,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002448586484801831
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.04031563564195608,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0022007717563761603
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.030622384698934416,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001718163099004444
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..3fb1eea25384c468c5c94a19a1b4273c570a9e34
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_4.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.20107349281952527,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.04214224677468119
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.049366458127839984,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002854014732476892
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.04923533037170826,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0024982212236816293
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.038633264035856554,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020222776724184797
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.0069147124408860524,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009836165108404464
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.0069063816404413315,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0008881448140462662
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.005146040515208341,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006133922293712356
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.043511782572989245,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002558689962342701
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.04427204057886875,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.002279980453794631
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.0341478013156332,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0018057433669060028
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.04448265410501512,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026386860903847256
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.04405637966133121,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0022677196418328066
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.03455201836586164,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018292045551711296
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ec598c6fc26465f9603bac2992ccaf0718d8db6f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_no-prompt-needed_5.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "bleu": 0.2315660064365379,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.03985185692641373
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_precision": 0.04883644295164218,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0027824290368789364
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_recall": 0.046722404599889716,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0023869495808978747
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge1_fmeasure": 0.03797730190196727,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0019584581823532746
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_precision": 0.006695738235568391,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009608876674204776
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_recall": 0.006591972710456567,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0008806024888411368
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rouge2_fmeasure": 0.005234740517822527,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0006778104688310928
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_precision": 0.04343865379951131,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002509403663109993
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_recall": 0.04203034225897826,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0021838302966321445
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeL_fmeasure": 0.0337124518984988,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0017510459198918948
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_precision": 0.04410239398251147,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002562032316728619
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_recall": 0.04154031189118946,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002165265517853104
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "no prompt needed",
+      "rougeLsum_fmeasure": 0.033884080328243846,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017631801582756708
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..c7cbc7a1a8b736c2e4a36b17cbb8499d13316516
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.49510337323177367,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01166526473007815
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.49510337323177367,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01166526473007815
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..940c7f9b1dcd139905c660dffb38397d17d86572
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.499455930359086,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665817258899168
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.499455930359086,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665817258899168
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d3bfba3330ce4307eee56cc8ce34ecdc361c666f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.5054406964091404,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665133500637066
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.5054406964091404,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665133500637066
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..36142ddddb853a8ad1b5759451396587de86073c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.515233949945593,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011660408257153638
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.515233949945593,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011660408257153638
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..592c784c5830bed0f25b5ed29304cfbf4cfd64c5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.514145810663765,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011661154475524836
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.514145810663765,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011661154475524836
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2bec3a4db43ca647630d65327a1c0ebaa6b16be9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_pick_correct_choice_index_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc": 0.5087051142546246,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011664055982032842
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "pick_correct_choice_index",
+      "acc_norm": 0.5087051142546246,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011664055982032842
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4036274ce18f4c3708fe62dc51956d29a8d48b87
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.4956474428726877,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665382144642394
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.4967355821545158,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01166557553076037
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a47e14fafd2f4eabab35c7397c4a475195988483
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.5092491838955386,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011663828032649187
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.5092491838955386,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011663828032649188
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c772df464edd4b41e55d3e61af316763b7a29798
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.5108813928182807,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011663061261117737
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665264730078145
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3eb6bfe681d6e77770f6e615cafbb9313b94c872
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.515233949945593,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011660408257153638
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.5070729053318824,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011664656918145945
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..3cd42453cccba9b84931ed148d370c09d18eee9c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.5114254624591947,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011662778026451675
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.501088139281828,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665796539540878
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..b0b540a2ae4e7d326fda5e9d7a76154acb52fc2b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_piqa_what_is_the_correct_ending_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc": 0.5048966267682263,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.011665264730078145
+    },
+    {
+      "task_name": "piqa",
+      "prompt_name": "what_is_the_correct_ending",
+      "acc_norm": 0.4972796517954298,
+      "dataset_path": "piqa",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.011665651503000732
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..27f3eb183c57fe63c8d601c0593463c858903b02
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.296,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014442734941575018
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.312,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014658474370509005
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a41539917ec9f8e4bd486aa8c4b38ab097519115
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.3,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014498627873361427
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.327,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014842213153411247
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..919e738e6dd5571b142a93a01b3f77def2cab937
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.299,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014484778521220473
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.338,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014965960710224494
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d45c04305d731ccd918aca376c2fabbf3fea55a7
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.301,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01451239503354315
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.321,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014770821817934637
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc8b42f8699e275680a808469a61a82f6207447c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.309,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01461960097720649
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.325,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014818724459095526
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..a49b870b237bd5d69ced1ca3a50921a7b7f4eca5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question-(Closed-Book)_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc": 0.303,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014539683710535257
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question (Closed Book)",
+      "acc_norm": 0.328,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014853842487270336
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9c853491d88be1d1597bcf779be3ad1bedc26657
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.561,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015701131345400767
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.493,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015817749561843574
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e4376b34c6257e06288d2fafe15ae7ef4580fd4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.578,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01562562511262066
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.543,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015760691590136388
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..9e955925f96e17083f23bdb0a80d27351cff8dc5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.563,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01569322392873038
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.536,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01577824302490459
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..13644b22bee70582b037ac2cd901f0722c154ca6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.567,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015676630912181334
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.543,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015760691590136384
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7eb5976e6dc4d3553ace805f372d3cea42d1fed
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.573,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01564978964446221
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.557,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015716169953204105
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb7d5bcf54884d9aa846e98f8c73f10d227317da
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Direct-Question_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc": 0.573,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015649789644462214
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Direct Question",
+      "acc_norm": 0.562,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.01569721001969469
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..150cf158aded4332011a2f448e01ee43ce15aa12
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.302,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01452608023545955
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.316,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014709193056057135
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..ff24623930330046a2589806f3667d42706ac078
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.32,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014758652303574874
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.343,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015019206922356953
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e5ff7ab35c338a88c5b046ac3ff922d8ddbfc962
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.33,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014876872027456729
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.35,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015090650341444233
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea0f7f33223c9c33f07c139607a670ef3d98cfcc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.338,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014965960710224482
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.37,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015275252316519357
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..218fc51814a6a816621845a4aeac5a6b2e3b6b54
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.345,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015039986742055235
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.346,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015050266127564446
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..52e98195468c763370487ac29eacac4ad281a33b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-(Closed-Book)_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc": 0.325,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014818724459095526
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice (Closed Book)",
+      "acc_norm": 0.34,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014987482264363935
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..69d6a9216815c429eba93993f8bda03c3f9f2887
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.353,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01512017260548369
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.343,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015019206922356951
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..757cc3c13f0e339adbf80f4b4bab985962f4d097
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.362,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015204840912919498
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.367,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015249378464171756
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..5c79ea5bc9a4f8a94b939136d87f51098a50eb82
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.374,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015308767369006358
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.38,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015356947477797572
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d90e3bb17ef196c3983ce4cb35d7c3752b12c4ad
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.38,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01535694747779757
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.398,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015486634102858917
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..be91bf39f75a46ec305808ec7ae187fbaef756e5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.37,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015275252316519366
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.394,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015459721957493382
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..4426d4770434714b2df81c0f04225209969d78ca
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice-Question-First_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc": 0.375,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015316971293620996
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice Question First",
+      "acc_norm": 0.374,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015308767369006363
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..2ed8be468e49505b16ffcf6eb6cf4200580fdc7a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.326,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.01483050720454104
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.343,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015019206922356953
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..813ad6bc1f7b629bbaf957e6af51ada1cdfef5fb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.329,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.014865395385928357
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.34,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.014987482264363937
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..36a7aaa9e8f4c65ad210ef150011c80a1d72b9e6
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.344,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015029633724408943
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.351,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015100563798316403
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fd4200a4f79e694e3bb054f2369798220679b550
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.362,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015204840912919496
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.373,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015300493622922812
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1827c7b40da7af52c850a4bae63d1f833fd87ea
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.356,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015149042659306623
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.36,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015186527932040127
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..2f51a26b52328fa36e96498311e00523f9da1997
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_sciq_Multiple-Choice_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc": 0.344,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_stderr": 0.015029633724408943
+    },
+    {
+      "task_name": "sciq",
+      "prompt_name": "Multiple Choice",
+      "acc_norm": 0.366,
+      "dataset_path": "sciq",
+      "dataset_name": null,
+      "subset": null,
+      "acc_norm_stderr": 0.015240612726405756
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..cbfbee331783dc9ecef6e2aa7eeaa0b629ba407c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.5056119722073757,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011561703928784335
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5125601282736505,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01155878357073797
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7582cb90c10f08f07cfcd8ca007d15f006244f9b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.49812934259754144,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011562351329083266
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5088188134687333,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011560633656952961
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..3aba9049cec09bb968a3a730cfad52d0dbcaa912
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.4949225013361839,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011561836054238776
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5173703901656868,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011555452669106635
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea94902ffb9748c6254079e16fbbe03f8f337d45
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.48850881881346875,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011559378273599118
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5082843399251737,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01156084507652571
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..02aef580606bdc854e15e103f5d353d4282061b5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.4965259219668626,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011562153149168298
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5163014430785676,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01155628548452156
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..98808405664c5e77b767ca919c1f4120c82a0778
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Answer-Given-options_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc": 0.49331908070550506,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011561400034509398
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Answer Given options",
+      "acc_norm": 0.5114911811865313,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011559378273599126
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..6f105c686398600b7aa279428283f2c866a65a05
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.518439337252806,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011554566910658103
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5456974879743453,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.0115140402455835
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e3e9b2d92f1983ab4886eef260fa6104695e883b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.5168359166221272,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011555875693960771
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5339390700160342,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01153576488164141
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..835070f0e9325bb64ad00b3489cdd6bb0a1d0e7f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.51309460181721,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011558466383367176
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5392838054516301,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011526690316014594
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..4b098609f1cbf38123c95dacf05653fcf62e7b1b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.5056119722073757,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.01156170392878433
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5275253874933191,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011544898473864576
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..511b95b82fa402ada1617b1d47b7218c229e171b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.5125601282736505,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011558783570737967
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5312667022982362,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011539803085637727
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..f10781b53afbdc180314a1a1a5ddbf1223382cab
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Choose-Story-Ending_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc": 0.5056119722073757,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.01156170392878433
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Choose Story Ending",
+      "acc_norm": 0.5275253874933191,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011544898473864574
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..59cd6c00d996b255b1cd835df39e8ebf8336e7a3
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_0.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..a8764b3a4b69032853f571b24fc2c92d85fc5559
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_1.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0622a0fa72dfae1645f030c9a7acc8193bb5b7d
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_2.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..0c8b7742037cb3a036805950dd6e390bf4f6af03
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_3.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e45d42123a6fc4b65c1621f1ecf4f9dd78995b34
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_4.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..ce5198198219fbb678ff4ba94ee68a25061891fc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Generate-Ending_5.json
@@ -0,0 +1,15 @@
+{
+  "results": [],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..501081eab3ef2b112b4e0865dbc5f27e80c9d4cc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.5008017103153394,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011562417388300206
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.5136290753607696,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011558135970599896
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f7fe001ea8861d18978946886c357b549b076221
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.4863709246392304,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011558135970599896
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.5104222340994121,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011559920087347773
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..005df77fbf5d951bd65e3b56a47cff2f1cee068f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.48583645109567075,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.01155779233130167
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.5056119722073757,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011561703928784332
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe0c72e091c1610fa90c5d904fefd1e20f2c4b23
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.4836985569214324,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011556285484521565
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.49706039551042225,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01156223242154194
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..252f6ecb65d9e6d7715c6c9d43314a8a1bc2b470
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.4826296098343132,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011555452669106632
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.501336183858899,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011562390964658753
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..0692ccc849d3b2d5ce9b46bc6d585e7254d18c08
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Novel-Correct-Ending_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc": 0.4890432923570283,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011559655791130734
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Novel Correct Ending",
+      "acc_norm": 0.5056119722073757,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011561703928784332
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..05a8b1d977326c20ebe3ea119ae6d08f328afab5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.5168359166221272,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011555875693960774
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.5376803848209514,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011529552555884568
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d87766fa870690c208e4cb5108e463e9021c19a8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.5120256547300909,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011559087533800689
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.5360769641902726,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.01153229486915312
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..d55bf6f3bba5fdd879bba29bfe06729201ee22bf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.5066809192944949,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011561400034509398
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.5264564404061999,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011546234813777412
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..23f6d4dd65051deef978a961e29753178cbc08e5
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.4922501336183859,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.01156104327886354
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.521111704970604,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011552120807053817
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..a03f6df6d5577d81e122ffdc2b0dce1358b2f2b8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.5018706574024586,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011562351329083271
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.5243185462319615,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011548748301487312
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..8202d6553a546e847a2e8f69ba5fc10fc18ea5ab
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_story_cloze_2016_Story-Continuation-and-Options_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc": 0.498663816141101,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_stderr": 0.011562390964658758
+    },
+    {
+      "task_name": "story_cloze_2016",
+      "prompt_name": "Story Continuation and Options",
+      "acc_norm": 0.5232495991448424,
+      "dataset_path": "story_cloze",
+      "dataset_name": "2016",
+      "subset": null,
+      "acc_norm_stderr": 0.011549925483927461
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..0a694344d0ffedeb45d23b302a6568aa39f4f9f9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.5270758122743683,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030052303463143706
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.5018050541516246,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030096267148976626
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3cd4d2e5027ce8a794fd17d9044f30d8cfec2d33
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..3663d33502b9b1400daf1369af8ed668bf29d961
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.5018050541516246,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030096267148976633
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc6c29eaa037e417f93a9ce38e4226605f0eea35
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030094698123239966
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..6b6667bcdf030fce91fcaa02fc9290f8d1de30b4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.48014440433212996,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.0300727231673172
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.4548736462093863,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.029973636495415252
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..aabecaedd818ac65c1850044a7f1fa0ff470a463
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_GPT-3-style_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc": 0.48736462093862815,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030086851767188564
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "GPT-3 style",
+      "acc_norm": 0.47653429602888087,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.03006330041190266
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..d225fec55b834bda8e7b28e84d9dbb976c78b47e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.516245487364621,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030080573208738064
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.5270758122743683,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030052303463143706
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c50b4744e0138f3370648b3bf32fca088c282d08
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030091559826331334
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..249c3a16a6f121fe04b0849da4261147a608fb40
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.4981949458483754,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030096267148976633
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.48736462093862815,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030086851767188564
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e063084645abccc85f287c7a62077d95dc96a09b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.4981949458483754,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030096267148976633
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.48375451263537905,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030080573208738064
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..7d31bad647bb410da9364de9dd7b4c30c54902fa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.5054151624548736,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.5054151624548736,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030094698123239966
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f1af1faecd31347d1e6fe77b15164a357f044fa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_MNLI-crowdsource_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc": 0.48375451263537905,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030080573208738064
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "MNLI crowdsource",
+      "acc_norm": 0.5018050541516246,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030096267148976626
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..e40dadd02b72a09d924e6e2bf9131cd452089cd0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.4368231046931408,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.029855247390314952
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.5090252707581228,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..e703fcdf7726eb2a4f0012314d07fbf9905c3a34
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030091559826331334
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6655ea47efb4a74e66b0da15de07f9a0b39012a9
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.48736462093862815,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030086851767188564
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.48375451263537905,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030080573208738064
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..9d9a464c96fa0c3b48b9d5814e5cd9188ad3d0e1
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.47653429602888087,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.03006330041190266
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.4548736462093863,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.029973636495415252
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..051caa339c35dae3cdcaa6951db5fdce724c218a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.4729241877256318,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030052303463143706
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030094698123239966
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..109c1ef301101d4497bbc578eaa4d0d3f95cc48f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_does-it-follow-that_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc": 0.4620938628158845,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030009848912529117
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "does it follow that",
+      "acc_norm": 0.4693140794223827,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030039730592197812
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..5f4dbceaf887e2911fef6b661c88747802299b85
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.5270758122743683,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030052303463143706
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d1e65683255e11dfed98179ebeb010e0ce5a5eb4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030091559826331334
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.48375451263537905,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030080573208738064
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1253553a0136e660470c0fa68bd029721091c2a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.47653429602888087,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.03006330041190266
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.47653429602888087,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.03006330041190266
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..e0206224a3255a1b651f6a5165e490703372672f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030094698123239966
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.4657039711191336,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030025579819366426
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..d03f864e69199aa94b2d5f13a9224f064c9094ae
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030091559826331327
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030094698123239966
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..77daccb67e81a32d7b4f139fad18ad4d4709ec57
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_guaranteed-true_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc": 0.4981949458483754,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030096267148976633
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "guaranteed true",
+      "acc_norm": 0.4981949458483754,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030096267148976626
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..31c702abc7f5a9a0f21749cf6fdd9616a68d443f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.5270758122743683,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030052303463143706
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.5270758122743683,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030052303463143706
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..c5d560fd9d406d27df990faa65b40425dc82d62f
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030091559826331334
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.49097472924187724,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..e097d1028b9db262063847cf90078b563a7cd18c
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.48014440433212996,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.0300727231673172
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.48014440433212996,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.0300727231673172
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ef9a6fdd33402cb04bcdbdc0f1c7c5f231db356
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.48375451263537905,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030080573208738064
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.49458483754512633,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030094698123239966
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..bd3843770ae224977cdf0483152b2649cf916afe
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.48014440433212996,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030072723167317194
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.5018050541516246,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030096267148976626
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..96dec73f77d66f9eb920c112a9b39884a7f14cd4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_superglue_rte_should-assume_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc": 0.4981949458483754,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_stderr": 0.030096267148976633
+    },
+    {
+      "task_name": "superglue_rte",
+      "prompt_name": "should assume",
+      "acc_norm": 0.5090252707581228,
+      "dataset_path": "super_glue",
+      "dataset_name": "rte",
+      "subset": null,
+      "acc_norm_stderr": 0.030091559826331334
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..9903239378a0e716fc8d1c06291e9db5e3f6d334
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.5059194948697711,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01405150083848581
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.4940805051302289,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014051500838485807
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..3d3fea9d786ca33e4e6bed1a95301444de3b04a8
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.49329123914759276,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014051220692330349
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.500394632991318,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052481306049516
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..89f17066ab1576d41abc2dcc859595ab336ce572
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.4877663772691397,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014048278820405621
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.5090765588003157,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014050170094497704
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..d6809aeb1700f3b11a34491bc46b798ac59d7d95
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.505130228887135,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014051745961790516
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.5074980268350434,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014050905521228571
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..4bd27de7c1c389abcc608d16d0721393d55b1b55
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.510655090765588,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.0140492945362904
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.5035516969218626,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052131146915852
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..4271c47252c903ee3bbdcc69e5207919c8aefbaa
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_Replace_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc": 0.505130228887135,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014051745961790516
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "Replace",
+      "acc_norm": 0.4964483030781373,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052131146915867
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..a64aa6ba2f9e6d0b6756b502e435aad4b85019ab
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.4980268350434096,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014052376259225636
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.5082872928176796,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014050555322824192
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..bdb473ef4d4370096fb582f9a099a128674c8091
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.4940805051302289,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01405150083848581
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.494869771112865,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014051745961790516
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca12f832f16de64831cf3c4e1d74fe5883308907
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.5035516969218626,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014052131146915845
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.500394632991318,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052481306049516
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ecd128b137677073e3614c235b6186bbb9acdf20
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.5059194948697711,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014051500838485807
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.5019731649565904,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052376259225629
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..22d49ce3cb014087c3b48784db293358173803bf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.526440410418311,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014032823874407224
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.5232833464877664,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014037241309573645
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..c8b4ca5e5f6b79f71ca3ed996b032960e12190f4
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_True-or-False_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc": 0.5122336227308603,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01404827882040562
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "True or False",
+      "acc_norm": 0.5090765588003157,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.0140501700944977
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..4a85e33627c34611cd212ce1e258065f3a5f7acf
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.4877663772691397,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01404827882040562
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.4846093133385951,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014045826789783665
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..b1889b63569ac23ad3f75523d15e3d1a275786cb
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.4877663772691397,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01404827882040562
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.49013417521704816,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014049749833367589
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..6fab831000a173b04d9be5a3375b3eb51ac4bdbd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.5201262825572218,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014041096664344327
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.4988161010260458,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052446290529019
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..7634100554da64ee77f766c5c006e5abbbb4d42b
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.5122336227308603,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01404827882040562
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.5035516969218626,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052131146915857
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..b555076e52fc60dea4772dc77cc40fd77758c02e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.5114443567482242,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014048804199859322
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.4980268350434096,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052376259225629
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3605411f147bc21c97b4f08378c244d2e281a904
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_does-underscore-refer-to_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc": 0.489344909234412,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014049294536290393
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "does underscore refer to",
+      "acc_norm": 0.49171270718232046,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014050555322824192
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..83c674f878a8b186efc361cab94a449db787909a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.4925019731649566,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01405090552122858
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.48855564325177586,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014048804199859325
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..d4b0bd41cc8abc8aca73cd825afe64f0e478ff04
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.500394632991318,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014052481306049516
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.4980268350434096,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.01405237625922564
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc1a64f51bd752f2299e0536e3fe9e429d04645e
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.5169692186266772,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01404439040161298
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.5027624309392266,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052271211616445
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..ebab98fd28eb98aee18ece3a387eb67aea465063
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.5122336227308603,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014048278820405616
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.5035516969218626,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052131146915857
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..07408e076d7a4621cc6ad2d74babf7621533a8fd
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.5098658247829518,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014049749833367596
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.489344909234412,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014049294536290403
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..5867113b9ed4b2a39090a2bc3990d3c4af2d4cc2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_stand-for_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc": 0.5090765588003157,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014050170094497712
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "stand for",
+      "acc_norm": 0.48303078137332284,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014044390401612972
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json
new file mode 100644
index 0000000000000000000000000000000000000000..7cc16519cff93b8b7d74f4aa791ed57ddda4a7dc
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_0.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.48855564325177586,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014048804199859329
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.4925019731649566,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.01405090552122858
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 0,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json
new file mode 100644
index 0000000000000000000000000000000000000000..f3a4c6f2c422abec442cd721c9c04f88bcb0cdb2
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_1.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.48539857932123126,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014046492383275835
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.48697711128650356,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014047718393997663
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json
new file mode 100644
index 0000000000000000000000000000000000000000..8fcd0496440da382426a09f9f8e4252fdf04387a
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_2.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.4925019731649566,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014050905521228573
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.500394632991318,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014052481306049516
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json
new file mode 100644
index 0000000000000000000000000000000000000000..61c795dd778d73abe8af83ffd43370a9d34e3593
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_3.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.5074980268350434,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.01405090552122858
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.4956590370955012,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.0140519560640769
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json
new file mode 100644
index 0000000000000000000000000000000000000000..e1399a738ee7c5b5244459d3e2ecc1d32e5966c0
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_4.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.500394632991318,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014052481306049516
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.49171270718232046,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014050555322824192
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json
new file mode 100644
index 0000000000000000000000000000000000000000..3e007baa53238eb90972d6e201306db138895232
--- /dev/null
+++ b/1b11b51b5/eval/slim.lm1-1b1-1b5_winogrande_underscore-refer-to_5.json
@@ -0,0 +1,34 @@
+{
+  "results": [
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc": 0.4988161010260458,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_stderr": 0.014052446290529019
+    },
+    {
+      "task_name": "winogrande",
+      "prompt_name": "underscore refer to",
+      "acc_norm": 0.49013417521704816,
+      "dataset_path": "winogrande",
+      "dataset_name": "winogrande_xl",
+      "subset": null,
+      "acc_norm_stderr": 0.014049749833367596
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..455573a4bffeffcac93748098bd2844b3dfc93a6
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.1715109274753528, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.02642539468126329}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.056271792717741465, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0020834882225830133}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.17970657292444336, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037758846666814077}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07392470782815987, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.00177901763739792}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.021453121863695197, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0010399844752529916}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07781374849374786, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002432084660851988}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03028468748562561, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010296918553042514}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.054104790772534245, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002018866888899467}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17534202443784297, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.00369978730480511}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07124999476464311, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016868036669108117}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.054126123805613165, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002026505118360271}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1747286304108634, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003660334571889786}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07115192816057488, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016843849706888723}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..a7d3642c924ab9c7d1f0a96e711d5af77010d86a
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.11694864689561944, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.010760323698374666}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.05492023407017857, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016326725952034475}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.17990254142984613, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037492925899754617}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07558454105004475, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017728413797174199}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02075424080508391, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009150263161295265}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.0760260739680381, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023774787682776578}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.02956013283155878, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009596682875711376}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05153123991260807, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014581185184364613}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17447931108768522, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036741060344485967}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07194166497021176, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001647469505361452}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.052002469160781814, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015071594845800093}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.17445084715749506, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0036515775079597516}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07220041477281516, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016608929228483667}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..fa9c608cf2a7d71ec94231ac80c4445ca07d1ddd
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.157083884873114, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.025447861405185956}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.059290722393941356, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019300973301455053}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.18222315758965466, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037052853928907043}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.07793163760494656, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0017891627195254817}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.021478064415916574, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009791043575277162}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.07687939137894265, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023619853957239805}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.029975159373482684, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009737894387487846}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05528292115358125, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017712704886265283}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17551310076734894, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036271666172633756}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07339358383221016, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016455203554569008}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05583469963213412, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0018018226490151468}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.17544849623130826, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0035888045371675796}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07381888878270101, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0016588962455865598}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..9ac3a45afc4af2ea08fe6c2b4aac30c18d338c46
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.19732756628191997, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.026658287671472842}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.06070458892233607, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017816786117538406}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.19093902196152424, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003753006790856248}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08257826480436961, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018872309614066823}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02314057822422759, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0009303717509439983}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.08094570128568006, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.002426051231637763}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.03236773058084414, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001033048999274749}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.05654666196831523, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001579449111979041}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.18383917337722214, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0036538299358506152}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07788575137246734, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001720896690999757}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.057097033183754045, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016220022129662945}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1838313736752301, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.003621428295981444}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.07823625277495454, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017358911503115172}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json
new file mode 100644
index 0000000000000000000000000000000000000000..d41361036c16d31e486359b23426f5e0b6792565
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "bleu": 0.14367082750773918, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01727398003487744}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_precision": 0.061364801134236466, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0019895158236849513}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_recall": 0.1852053079126511, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0037095859419792}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge1_fmeasure": 0.08023946961419831, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0018599842014871484}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_precision": 0.02155110915099608, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.000944250904374556}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_recall": 0.0760367527800932, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0023514862121833038}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rouge2_fmeasure": 0.0300231413049405, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009672037164958253}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_precision": 0.056807969707911, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0017323813715249387}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_recall": 0.17835942307556787, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003607251708504229}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeL_fmeasure": 0.07552378160245939, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016866442023514398}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_precision": 0.05708185625481882, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0017532405619586147}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_recall": 0.1779104329111164, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0035688476761420685}, {"task_name": "GEM/web_nlg_en", "prompt_name": "PALM_prompt", "rougeLsum_fmeasure": 0.0756658622438174, "fixed_answer_choice_list": null, "dataset_path": "GEM/web_nlg", "dataset_name": "en", "subset": null, "prompt_id": "3e41305c-5461-4cf3-853d-8a6fb5747623", "prompt_jinja": "I will verbalize an abstract representation of a sentence in natural language. To do so, I will first show the representation and then the natural language. The text needs to include all of the information in the representation.\n\n{{input | join(\", \")}} {% for i in references %}\n  ||| {{ i }} \n{% endfor %}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017015828692780743}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..2538e3b811064b22a8d9e0a853f5e10528ca81ef
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.11243090334580715, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0015900812709383934}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.11715651177590802, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.001696871084875322}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.10058251758979377, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0012559808866013141}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006128533869165112, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0004018707853940709}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.007425968853967204, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0005962348292070988}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.005682283734931176, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00035696819267689325}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09041052243361407, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012316877506636582}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.09424789347821856, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0013396373126375609}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08035070126772184, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0009302451261276216}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.10789218625357785, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0015108158731484223}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11248330566212135, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016035045599026544}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09651169925228414, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0011859501014305082}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.4868015971525538, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08360711580226779}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..67898144b93d7e0ed84369336699e4474ba8a40c
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.11992316681250555, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0017675704654899571}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.13948810001589107, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002058649220725351}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.11194017044626671, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0014428492038973415}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.010522981314992496, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006069518989281641}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.014385796237686857, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008041815234389601}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.010243977830737588, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000493265280559601}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.09634769905116539, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0013910007655887457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.11364453759174203, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0016847934987669815}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.0897361308600908, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0010822617010161773}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.11382504495028803, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0016623075783859457}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.1321927492875554, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0019214290897240306}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.10607719694851259, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0013391525579239643}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.7583577300885451, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0594734221378448}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..d035fbe97ca210ac392442428a6e6c722362a144
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.10821789424681753, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002096638300657934}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.12472730510559357, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002322606633118786}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.09784665332111649, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0016192055617561215}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.01129846864454297, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0006277146654822384}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.015276009798409365, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0008518486109567291}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.010690797458122308, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0005103079982659685}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.08908785284327644, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001705722903459829}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.10389230433583506, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0019161308146752112}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.08028298986501597, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0012500164736239873}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.10264516065485063, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0019926607627734193}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.11786995367940563, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0021718471770820493}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.09248642582400225, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015079304696231108}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.8537954880656907, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.10033840689595877}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..a2f3eba1ca4dd350ba0554cc729b3fb6ee2a1132
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.04157996091862478, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018032030088400309}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.044926460912185065, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0018528180948563747}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.03524490990351491, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0013594404865196162}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.006050502360776094, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0005836053795733668}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.007180836611458644, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0006625116906510873}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.004931876443326115, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00038095104223179015}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.03456113363014217, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001498234658219552}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.0375951446671367, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0015455223646234392}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.029054409914181435, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.001089301348325835}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.03887702645834208, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.001695261016406579}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.04140726881716476, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0016967910317344356}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.03263081498138409, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0012475593919046596}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 0.11289573429538445, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.01845710657006998}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..62e8a3600529a46767c66212c13a962b5a37d0f1
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_precision": 0.008254948637118429, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0009459136809570537}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_recall": 0.007562240110686054, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0008636106012690946}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge1_fmeasure": 0.006160737725223056, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0006512100088223974}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_precision": 0.0012474030450093298, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00025491125834316595}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_recall": 0.0013793630128949251, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00032763950112405063}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rouge2_fmeasure": 0.0009425275750514962, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00018272575729318913}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_precision": 0.0067843630893508035, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0007756800649250622}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_recall": 0.006224422456967982, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0007102549709126986}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeL_fmeasure": 0.00501321248753611, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005224006471877824}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_precision": 0.007737895530410426, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0008942014082809357}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_recall": 0.006995766927183111, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0007965814705828734}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "rougeLsum_fmeasure": 0.005755424216995409, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0006110349341052841}, {"task_name": "GEM/wiki_lingua_en", "prompt_name": "tldr_en", "bleu": 2.1707923159701563e-10, "fixed_answer_choice_list": null, "dataset_path": "GEM/wiki_lingua", "dataset_name": "en", "subset": null, "prompt_id": "d3c5baa3-5e37-46f8-b1b2-5b834181c9da", "prompt_jinja": "{{source}}\n\nTL;DR in English: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.2012188612881837e-09}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..3768f1d7e7381ee75ca034352c3c08e468baace8
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 3.741207237448518, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.12328594100727748}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.33565838168296036, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002543739891022053}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.27743476294626046, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022564269895565987}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.28858251510563476, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.001991339721709451}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.08431013375181877, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0015876617488378608}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.07011453690760112, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001335315815509775}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.07273933980217488, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0013272082416632288}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.24464584594762856, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0020076748803741955}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.20102123374490308, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017273862409986716}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.20937991809553785, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015406451796370017}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2774307167772055, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0023056419978161836}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.22843062477167406, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.001987724352272349}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.23796164791282326, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001803514609873287}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..51019531d5673181a1e5c1441afb0875023a306f
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 4.290890219148563, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.11797671966393032}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.3459713718406746, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002718482058174072}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.28815608021642963, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022672787761088384}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.2978425731750224, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020372027980421125}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.09498590594242383, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0017360545885379483}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.07791865939566535, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014093737049867404}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.08070496923908829, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.001400104340028372}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2542525441295202, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0021937081691301894}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2106927728428706, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0017758508939911513}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.21768665558633662, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0015933877730589948}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2862920446603683, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002475870560064218}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.2376715264121337, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002037562231835293}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.24573333528025115, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018602140974894146}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..2aa32fcfe5ac4089b8b24ac1ed95fddeb0a9107f
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 4.732867354096951, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.15404543486723934}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.35442414475500134, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002731868155718904}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.29373594326712926, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.002294258375803093}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.30378389260311006, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0020385063291000773}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.10512573374019807, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0018530498330245938}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.08588813414618772, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0014835043277026077}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.08882317107094179, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0014555231709394456}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2649706992736119, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0022811580211275924}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.218297790406449, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.001833736502924904}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2256061529203063, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016287697579368406}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.2972954010894401, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.002533132838252456}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.24506290855487275, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002062388974631323}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.25364769619114425, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018650391381006888}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..978fe23fd495e5633cb9c90d337eb2d8c6a82e37
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 5.210131054318675, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.16155247866216102}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.36395994345002086, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.002850604700096735}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.29667153746092684, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022956768105348727}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.30893188850915426, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002078050469669259}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.1140043958983537, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00202343696017089}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.09103549434582939, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001530348953726892}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.09490796525239133, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015347079760504483}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2740532547932312, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0024194755484310826}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.2217823173210715, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018382921326583175}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.23092897642137852, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016828405073314597}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.30628692608188124, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026427962890604785}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.24850367361793804, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002082869893324162}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.2590295247906419, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0019213450001221285}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..641fc75279c31f6a570bc56faa36aed96d682c99
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "bleu": 5.3388680014012015, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.13008777046551853}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_precision": 0.36808074646470906, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0028756888288344754}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_recall": 0.2997544520993915, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0022774797385096417}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge1_fmeasure": 0.3120715598551173, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002049220094979265}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_precision": 0.11888183945969535, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0020436399790904566}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_recall": 0.09438735759794968, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.00155057405248203}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rouge2_fmeasure": 0.09841408639827368, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0015376685877858227}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_precision": 0.2806218677955488, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.002465381284480861}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_recall": 0.22659913710533197, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0018404855639564178}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeL_fmeasure": 0.2360616277086933, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016771641427817245}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_precision": 0.3119441169782797, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0026747982066311175}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_recall": 0.2526371639744921, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0020647828708917995}, {"task_name": "e2e_nlg_cleaned", "prompt_name": "generate_text_restaurant", "rougeLsum_fmeasure": 0.26331410062448574, "fixed_answer_choice_list": null, "dataset_path": "e2e_nlg_cleaned", "dataset_name": null, "subset": null, "prompt_id": "1acabbc3-c9b9-4624-a684-29faeccff46f", "prompt_jinja": "Given the following data about a restaurant:\n{% for feature in meaning_representation.split(\"]\") %} {% set key = feature.split(\"[\")[0].replace(\",\",\"\") %} {% set value = feature.replace(\",\",\"\").replace(key+\"[\", '''') %}\n{% if value != \"\" %} {{key}} : {{value}} {% endif %}\n{%- endfor %}\nGenerate some text about this restaurant. ||| {{human_reference}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0018958344026201047}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d0fbe18e90396d27140b1c66332f7281a269141
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.09734130580010827, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.001771813879632219}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.23205596880640392, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003968640338152418}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.1347806025835107, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002364452110807362}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.013809019824869628, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007842126128288541}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.03384878468635188, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0018246616684185503}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01937279969544618, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0010767896556060932}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07877507926978208, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.001317195018620984}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18918199375900235, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0029962553876845713}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10920408732481979, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017466454015791766}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.07655704256204823, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014263611561085394}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.18366939517842565, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0032532823145414863}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.10609898226051803, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.001907016281645563}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.7556443558763775, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08356354268542657}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 1, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..6d98b2a4a5a9a862a77064dff9fc8a4f0dceaf70
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08876555984023238, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0016308308833255663}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.21454525270713967, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0035861819885292467}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.12357434836180202, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.002149017836427172}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011772595475451241, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007152713643140575}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02914764125397616, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.001651586800843389}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.01649554595160008, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.000971551127185768}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07425164547734261, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0012476815962285165}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.18105447617675274, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.002824838892161731}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.10362842140926162, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016508490182790031}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06960818379765893, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0013153158283565624}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.17002840065420005, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002984809197834264}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09716065684449275, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017475396647593806}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.5986931718476021, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08317206014621986}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 2, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..e87892125dd863e3c30b47a5e36b29da0468932b
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.08457756400468115, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018422703241089745}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.195876410222803, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0038335449412391716}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.11439654244933288, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0022963595160460466}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.011624998600318197, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.0007516865101337179}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.02749715301034778, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0016328340556882883}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0159331168872316, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.0009893346680723053}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.07194115142590224, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0014722611772001854}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.167560514280581, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.003062295835710636}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0973492647648112, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0017981023986070029}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.06768573146684626, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014754231743952838}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.15747481060321722, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0030260328022856386}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.09140331616988624, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0017813872559020623}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.6437348489109824, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.08105590125094665}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 3, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json
new file mode 100644
index 0000000000000000000000000000000000000000..2c9ba787c3066fc4eab2c899045e478c43eb569d
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.026491777281042975, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0018068087051168196}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.0494491413339096, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.003027387600352431}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.03189372524424132, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.0019572020109923736}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.0034382794036798173, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00047491954770546336}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.006467065967569708, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 0.0007276044578978292}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.00410550208994294, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 0.00047252936288694365}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.02246733493087934, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0015021222768415922}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.042219853518797514, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0025299073135696837}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.027101703724227193, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0016162858861853424}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.02108679945636943, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0014787824836100922}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.03900351353665605, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.002373549357207589}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.025143845266448906, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0015404507929384814}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 0.22579456084743735, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 0.0510480477202509}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 4, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..895ebf91013096e30d34858add02a90d1e98b532
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/agg.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1 @@
+{"results": [{"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_precision": 0.0029325830197336608, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_precision_stderr": 0.0007845716748468843}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_recall": 0.002212272201507509, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_recall_stderr": 0.0006115672097018799}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge1_fmeasure": 0.0024394449960508753, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge1_fmeasure_stderr": 0.000649872781418437}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_precision": 0.00017152658662092626, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_precision_stderr": 0.00012780446523672458}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_recall": 0.00010681142756614455, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_recall_stderr": 7.755982479808459e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rouge2_fmeasure": 0.0001313177797301129, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rouge2_fmeasure_stderr": 9.625094432461928e-05}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_precision": 0.0023349726205139387, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_precision_stderr": 0.0006284522046757939}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_recall": 0.0017335948670290622, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_recall_stderr": 0.0004670903743357543}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeL_fmeasure": 0.0019268220007723433, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeL_fmeasure_stderr": 0.0005091814465631191}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_precision": 0.00249935226602566, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_precision_stderr": 0.0006799933367648228}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_recall": 0.0019134028298558926, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_recall_stderr": 0.0005458175188244193}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "rougeLsum_fmeasure": 0.002087647218353957, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "rougeLsum_fmeasure_stderr": 0.0005649060898529495}, {"task_name": "gem_xsum", "prompt_name": "article_DOC_summary", "bleu": 1.5544568735399087e-43, "fixed_answer_choice_list": null, "dataset_path": "GEM/xsum", "dataset_name": null, "subset": "", "prompt_id": "a8d4ecfa-c944-44d5-878c-04fd5db59e64", "prompt_jinja": "Article: {{document}}\n\nSummary: ||| {{target}}", "prompt_original_task": true, "comment": "", "bleu_stderr": 1.081029400958023e-37}], "config": {"model": "hf-causal", "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16", "task_args": "", "num_fewshot": 5, "batch_size": 16, "device": "cuda", "use_cache": false, "limit": 3000, "bootstrap_iters": 10, "seed": 1234}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..719e515b4d7ef0246292e123f6f701552838ecad
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4beca8004ceb8e336d3a0a564880ae2eadab0f58aaae9789565497ffe9f691b8
+size 4833131
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7bffd3ebbce22b8655dd386786533df5bb4a015a
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c529032adcec99f974bb5c6f3905f16479dbfad9ab2a15a694c8f621b3b375b8
+size 5689060
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9135a290ef2d29cc1503ebb6a83524f1dac01294
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de5f0c7103e3804f5bd1fbdc39aff7504d146b2e3f03f7c769392a518c7eb6ea
+size 6587020
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..200394528878441abdfd977257d212216d28cf7c
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee52af6732d0b2f117b737f619822797403759104fad1e690771d474a627027
+size 7493578
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5c58aab39ee204027b740958d86ea1728e0990c4
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7667fa968057805230ee630f786c66e4dd78871d3a3ae91282a8cbb02cade4
+size 8385909
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7d05550746292916c2b5ecfe1f9ad84630dcaf92
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7c2968e05e17c01dcc694eeff1bb51e58b95b716b731053f7abecb706df5bd2
+size 13078036
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..39f679d7a46726e46ff75e955be1d9d11d863ca2
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d71d3faf5125085e93497f6d6355a49ee8f68ae85370788cf7ca7949fbf2c782
+size 18732339
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1f1deff1adddd769acd240d7102ef6315c8b05cb
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:142ac5d1ac7ad04e62a849559d590c9f75a30b9900ba218ed6dc64ddb9eb8bcf
+size 24200516
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..826872ffe5a1e3a2bafb6821c73c43359ce9f57d
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bfcebff66bba230b04679da93f48a27c27ed16bb2c67f1e36293249b762ff87
+size 29428383
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1e2d03101485ede3ecad753795a0469a25e6fd26
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e68d565fe0a21c975c5d6912e7c920fc81b942d164351ceec081d0d837a22427
+size 34790197
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8ae0e0223f3e6c705014b1c3a3dd71e2b45ad8ff
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cc7ca9647deaf9a38eda620a837dfe24aaa26ec69002753048a4e071b3ea8ef
+size 5032535
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..396d7e3dd4b2d4ce1aee80e9b2b76efb5d0c75b6
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e67f9fe0d8ad7f410a3c5857edb2d46d719af5562472a5eae604837c60b89b7
+size 6126680
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4742e081e5c1c6c4f288f871f1e476215fd45601
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a203dd73aa71ba498c8b9157e913df0f47066bc999e5c8dbdf6bfe0fd0a92e3a
+size 7208113
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9f1f4555ad7349172be9140c1517bf0ee3b6b05a
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7fdfe834714a1f37c3acd8210339efe9e5e65356502cadb1ba53b74ac02d3ab
+size 8283999
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..93ee4cc2c935987d4694d1bbd56b4f3b745e8537
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872d0f7917872468ca6edb25fe3c1cfa3d653502327a36dbee8f0d5158acd8ed
+size 9372290
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..86da18f4103127227cf480a121fb943d8b230b03
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:643cb8d930e2f9e5c6c345a345461f66ffb9d4e14b1eb7de2a7aa57cfcbec6f7
+size 5095180
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..baaf1c10692413ec6e68a49d37af27d44f3c3680
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c501b45d38716b40434a3716b7857ece9dd2471f5df369bdcc76d0fa6edccd7
+size 7374512
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c2025a79a2618c154d22cdb70af9ee995d4bb3c0
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcca071a30ea099742057055965eaab6598b24f46942fc0a934d72c90750693a
+size 9640872
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9449b6246d1ae15ea36ccb7a4076c706acbe00de
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:376980008e82c5905383c95f51120b493b97bfa068a23db2670796bbdf7434eb
+size 11670288
diff --git a/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..e9902e963d996f19599c36ad7b128fa5bc3622df
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/examples.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.jsonl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ee8940f1b31c4f036581b815cf9f86db7ceb84a82d39d807de754aaef91098b
+size 13897406
diff --git a/1b11b51b5/evaluation/generation/merged_1shots.csv b/1b11b51b5/evaluation/generation/merged_1shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..eee2652b8872711aa5bed4a1d80e6cde5ca5ff94
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_1shots.csv
@@ -0,0 +1,5 @@
+dataset,prompt,metric,value
+e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.07273933980217488
+gem_xsum,article_DOC_summary,rouge2_fmeasure,0.01937279969544618
+web_nlg_en,PALM_prompt,rouge2_fmeasure,0.03028468748562561
+wiki_lingua_en,tldr_en,rouge2_fmeasure,0.005682283734931176
diff --git a/1b11b51b5/evaluation/generation/merged_1shots.json b/1b11b51b5/evaluation/generation/merged_1shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..35308c81c2a6959552956f235a7704f2a6993db0
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_1shots.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"PALM_prompt": {"bleu": 0.1715109274753528, "bleu_stderr": 0.02642539468126329, "rouge1_fmeasure": 0.07392470782815987, "rouge1_fmeasure_stderr": 0.00177901763739792, "rouge1_precision": 0.056271792717741465, "rouge1_precision_stderr": 0.0020834882225830133, "rouge1_recall": 0.17970657292444336, "rouge1_recall_stderr": 0.0037758846666814077, "rouge2_fmeasure": 0.03028468748562561, "rouge2_fmeasure_stderr": 0.0010296918553042514, "rouge2_precision": 0.021453121863695197, "rouge2_precision_stderr": 0.0010399844752529916, "rouge2_recall": 0.07781374849374786, "rouge2_recall_stderr": 0.002432084660851988, "rougeL_fmeasure": 0.07124999476464311, "rougeL_fmeasure_stderr": 0.0016868036669108117, "rougeL_precision": 0.054104790772534245, "rougeL_precision_stderr": 0.002018866888899467, "rougeL_recall": 0.17534202443784297, "rougeL_recall_stderr": 0.00369978730480511, "rougeLsum_fmeasure": 0.07115192816057488, "rougeLsum_fmeasure_stderr": 0.0016843849706888723, "rougeLsum_precision": 0.054126123805613165, "rougeLsum_precision_stderr": 0.002026505118360271, "rougeLsum_recall": 0.1747286304108634, "rougeLsum_recall_stderr": 0.003660334571889786}}, "GEM/wiki_lingua_en": {"tldr_en": {"bleu": 0.4868015971525538, "bleu_stderr": 0.08360711580226779, "rouge1_fmeasure": 0.10058251758979377, "rouge1_fmeasure_stderr": 0.0012559808866013141, "rouge1_precision": 0.11243090334580715, "rouge1_precision_stderr": 0.0015900812709383934, "rouge1_recall": 0.11715651177590802, "rouge1_recall_stderr": 0.001696871084875322, "rouge2_fmeasure": 0.005682283734931176, "rouge2_fmeasure_stderr": 0.00035696819267689325, "rouge2_precision": 0.006128533869165112, "rouge2_precision_stderr": 0.0004018707853940709, "rouge2_recall": 0.007425968853967204, "rouge2_recall_stderr": 0.0005962348292070988, "rougeL_fmeasure": 0.08035070126772184, "rougeL_fmeasure_stderr": 0.0009302451261276216, "rougeL_precision": 0.09041052243361407, "rougeL_precision_stderr": 0.0012316877506636582, "rougeL_recall": 0.09424789347821856, "rougeL_recall_stderr": 0.0013396373126375609, "rougeLsum_fmeasure": 0.09651169925228414, "rougeLsum_fmeasure_stderr": 0.0011859501014305082, "rougeLsum_precision": 0.10789218625357785, "rougeLsum_precision_stderr": 0.0015108158731484223, "rougeLsum_recall": 0.11248330566212135, "rougeLsum_recall_stderr": 0.0016035045599026544}}, "e2e_nlg_cleaned": {"generate_text_restaurant": {"bleu": 3.741207237448518, "bleu_stderr": 0.12328594100727748, "rouge1_fmeasure": 0.28858251510563476, "rouge1_fmeasure_stderr": 0.001991339721709451, "rouge1_precision": 0.33565838168296036, "rouge1_precision_stderr": 0.002543739891022053, "rouge1_recall": 0.27743476294626046, "rouge1_recall_stderr": 0.0022564269895565987, "rouge2_fmeasure": 0.07273933980217488, "rouge2_fmeasure_stderr": 0.0013272082416632288, "rouge2_precision": 0.08431013375181877, "rouge2_precision_stderr": 0.0015876617488378608, "rouge2_recall": 0.07011453690760112, "rouge2_recall_stderr": 0.001335315815509775, "rougeL_fmeasure": 0.20937991809553785, "rougeL_fmeasure_stderr": 0.0015406451796370017, "rougeL_precision": 0.24464584594762856, "rougeL_precision_stderr": 0.0020076748803741955, "rougeL_recall": 0.20102123374490308, "rougeL_recall_stderr": 0.0017273862409986716, "rougeLsum_fmeasure": 0.23796164791282326, "rougeLsum_fmeasure_stderr": 0.001803514609873287, "rougeLsum_precision": 0.2774307167772055, "rougeLsum_precision_stderr": 0.0023056419978161836, "rougeLsum_recall": 0.22843062477167406, "rougeLsum_recall_stderr": 0.001987724352272349}}, "gem_xsum": {"article_DOC_summary": {"bleu": 0.7556443558763775, "bleu_stderr": 0.08356354268542657, "rouge1_fmeasure": 0.1347806025835107, "rouge1_fmeasure_stderr": 0.002364452110807362, "rouge1_precision": 0.09734130580010827, "rouge1_precision_stderr": 0.001771813879632219, "rouge1_recall": 0.23205596880640392, "rouge1_recall_stderr": 0.003968640338152418, "rouge2_fmeasure": 0.01937279969544618, "rouge2_fmeasure_stderr": 0.0010767896556060932, "rouge2_precision": 0.013809019824869628, "rouge2_precision_stderr": 0.0007842126128288541, "rouge2_recall": 0.03384878468635188, "rouge2_recall_stderr": 0.0018246616684185503, "rougeL_fmeasure": 0.10920408732481979, "rougeL_fmeasure_stderr": 0.0017466454015791766, "rougeL_precision": 0.07877507926978208, "rougeL_precision_stderr": 0.001317195018620984, "rougeL_recall": 0.18918199375900235, "rougeL_recall_stderr": 0.0029962553876845713, "rougeLsum_fmeasure": 0.10609898226051803, "rougeLsum_fmeasure_stderr": 0.001907016281645563, "rougeLsum_precision": 0.07655704256204823, "rougeLsum_precision_stderr": 0.0014263611561085394, "rougeLsum_recall": 0.18366939517842565, "rougeLsum_recall_stderr": 0.0032532823145414863}}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/merged_2shots.csv b/1b11b51b5/evaluation/generation/merged_2shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..bee1bf9c713b75c6b46b76435551c98da6dad5c3
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_2shots.csv
@@ -0,0 +1,5 @@
+dataset,prompt,metric,value
+e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.08070496923908829
+gem_xsum,article_DOC_summary,rouge2_fmeasure,0.01649554595160008
+web_nlg_en,PALM_prompt,rouge2_fmeasure,0.02956013283155878
+wiki_lingua_en,tldr_en,rouge2_fmeasure,0.010243977830737588
diff --git a/1b11b51b5/evaluation/generation/merged_2shots.json b/1b11b51b5/evaluation/generation/merged_2shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..e525f63a6745801c373c9d3dcd1d207bb33cff9b
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_2shots.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"PALM_prompt": {"bleu": 0.11694864689561944, "bleu_stderr": 0.010760323698374666, "rouge1_fmeasure": 0.07558454105004475, "rouge1_fmeasure_stderr": 0.0017728413797174199, "rouge1_precision": 0.05492023407017857, "rouge1_precision_stderr": 0.0016326725952034475, "rouge1_recall": 0.17990254142984613, "rouge1_recall_stderr": 0.0037492925899754617, "rouge2_fmeasure": 0.02956013283155878, "rouge2_fmeasure_stderr": 0.0009596682875711376, "rouge2_precision": 0.02075424080508391, "rouge2_precision_stderr": 0.0009150263161295265, "rouge2_recall": 0.0760260739680381, "rouge2_recall_stderr": 0.0023774787682776578, "rougeL_fmeasure": 0.07194166497021176, "rougeL_fmeasure_stderr": 0.001647469505361452, "rougeL_precision": 0.05153123991260807, "rougeL_precision_stderr": 0.0014581185184364613, "rougeL_recall": 0.17447931108768522, "rougeL_recall_stderr": 0.0036741060344485967, "rougeLsum_fmeasure": 0.07220041477281516, "rougeLsum_fmeasure_stderr": 0.0016608929228483667, "rougeLsum_precision": 0.052002469160781814, "rougeLsum_precision_stderr": 0.0015071594845800093, "rougeLsum_recall": 0.17445084715749506, "rougeLsum_recall_stderr": 0.0036515775079597516}}, "GEM/wiki_lingua_en": {"tldr_en": {"bleu": 0.7583577300885451, "bleu_stderr": 0.0594734221378448, "rouge1_fmeasure": 0.11194017044626671, "rouge1_fmeasure_stderr": 0.0014428492038973415, "rouge1_precision": 0.11992316681250555, "rouge1_precision_stderr": 0.0017675704654899571, "rouge1_recall": 0.13948810001589107, "rouge1_recall_stderr": 0.002058649220725351, "rouge2_fmeasure": 0.010243977830737588, "rouge2_fmeasure_stderr": 0.000493265280559601, "rouge2_precision": 0.010522981314992496, "rouge2_precision_stderr": 0.0006069518989281641, "rouge2_recall": 0.014385796237686857, "rouge2_recall_stderr": 0.0008041815234389601, "rougeL_fmeasure": 0.0897361308600908, "rougeL_fmeasure_stderr": 0.0010822617010161773, "rougeL_precision": 0.09634769905116539, "rougeL_precision_stderr": 0.0013910007655887457, "rougeL_recall": 0.11364453759174203, "rougeL_recall_stderr": 0.0016847934987669815, "rougeLsum_fmeasure": 0.10607719694851259, "rougeLsum_fmeasure_stderr": 0.0013391525579239643, "rougeLsum_precision": 0.11382504495028803, "rougeLsum_precision_stderr": 0.0016623075783859457, "rougeLsum_recall": 0.1321927492875554, "rougeLsum_recall_stderr": 0.0019214290897240306}}, "e2e_nlg_cleaned": {"generate_text_restaurant": {"bleu": 4.290890219148563, "bleu_stderr": 0.11797671966393032, "rouge1_fmeasure": 0.2978425731750224, "rouge1_fmeasure_stderr": 0.0020372027980421125, "rouge1_precision": 0.3459713718406746, "rouge1_precision_stderr": 0.002718482058174072, "rouge1_recall": 0.28815608021642963, "rouge1_recall_stderr": 0.0022672787761088384, "rouge2_fmeasure": 0.08070496923908829, "rouge2_fmeasure_stderr": 0.001400104340028372, "rouge2_precision": 0.09498590594242383, "rouge2_precision_stderr": 0.0017360545885379483, "rouge2_recall": 0.07791865939566535, "rouge2_recall_stderr": 0.0014093737049867404, "rougeL_fmeasure": 0.21768665558633662, "rougeL_fmeasure_stderr": 0.0015933877730589948, "rougeL_precision": 0.2542525441295202, "rougeL_precision_stderr": 0.0021937081691301894, "rougeL_recall": 0.2106927728428706, "rougeL_recall_stderr": 0.0017758508939911513, "rougeLsum_fmeasure": 0.24573333528025115, "rougeLsum_fmeasure_stderr": 0.0018602140974894146, "rougeLsum_precision": 0.2862920446603683, "rougeLsum_precision_stderr": 0.002475870560064218, "rougeLsum_recall": 0.2376715264121337, "rougeLsum_recall_stderr": 0.002037562231835293}}, "gem_xsum": {"article_DOC_summary": {"bleu": 0.5986931718476021, "bleu_stderr": 0.08317206014621986, "rouge1_fmeasure": 0.12357434836180202, "rouge1_fmeasure_stderr": 0.002149017836427172, "rouge1_precision": 0.08876555984023238, "rouge1_precision_stderr": 0.0016308308833255663, "rouge1_recall": 0.21454525270713967, "rouge1_recall_stderr": 0.0035861819885292467, "rouge2_fmeasure": 0.01649554595160008, "rouge2_fmeasure_stderr": 0.000971551127185768, "rouge2_precision": 0.011772595475451241, "rouge2_precision_stderr": 0.0007152713643140575, "rouge2_recall": 0.02914764125397616, "rouge2_recall_stderr": 0.001651586800843389, "rougeL_fmeasure": 0.10362842140926162, "rougeL_fmeasure_stderr": 0.0016508490182790031, "rougeL_precision": 0.07425164547734261, "rougeL_precision_stderr": 0.0012476815962285165, "rougeL_recall": 0.18105447617675274, "rougeL_recall_stderr": 0.002824838892161731, "rougeLsum_fmeasure": 0.09716065684449275, "rougeLsum_fmeasure_stderr": 0.0017475396647593806, "rougeLsum_precision": 0.06960818379765893, "rougeLsum_precision_stderr": 0.0013153158283565624, "rougeLsum_recall": 0.17002840065420005, "rougeLsum_recall_stderr": 0.002984809197834264}}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/merged_3shots.csv b/1b11b51b5/evaluation/generation/merged_3shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..bd204b7fd036c6c8d6258e51092a7c1b6b51d9cf
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_3shots.csv
@@ -0,0 +1,5 @@
+dataset,prompt,metric,value
+e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.08882317107094179
+gem_xsum,article_DOC_summary,rouge2_fmeasure,0.0159331168872316
+web_nlg_en,PALM_prompt,rouge2_fmeasure,0.029975159373482684
+wiki_lingua_en,tldr_en,rouge2_fmeasure,0.010690797458122308
diff --git a/1b11b51b5/evaluation/generation/merged_3shots.json b/1b11b51b5/evaluation/generation/merged_3shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..94e80f0b4c1039edaf7cab17135c4b4fa24d6f75
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_3shots.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"PALM_prompt": {"bleu": 0.157083884873114, "bleu_stderr": 0.025447861405185956, "rouge1_fmeasure": 0.07793163760494656, "rouge1_fmeasure_stderr": 0.0017891627195254817, "rouge1_precision": 0.059290722393941356, "rouge1_precision_stderr": 0.0019300973301455053, "rouge1_recall": 0.18222315758965466, "rouge1_recall_stderr": 0.0037052853928907043, "rouge2_fmeasure": 0.029975159373482684, "rouge2_fmeasure_stderr": 0.0009737894387487846, "rouge2_precision": 0.021478064415916574, "rouge2_precision_stderr": 0.0009791043575277162, "rouge2_recall": 0.07687939137894265, "rouge2_recall_stderr": 0.0023619853957239805, "rougeL_fmeasure": 0.07339358383221016, "rougeL_fmeasure_stderr": 0.0016455203554569008, "rougeL_precision": 0.05528292115358125, "rougeL_precision_stderr": 0.0017712704886265283, "rougeL_recall": 0.17551310076734894, "rougeL_recall_stderr": 0.0036271666172633756, "rougeLsum_fmeasure": 0.07381888878270101, "rougeLsum_fmeasure_stderr": 0.0016588962455865598, "rougeLsum_precision": 0.05583469963213412, "rougeLsum_precision_stderr": 0.0018018226490151468, "rougeLsum_recall": 0.17544849623130826, "rougeLsum_recall_stderr": 0.0035888045371675796}}, "GEM/wiki_lingua_en": {"tldr_en": {"bleu": 0.8537954880656907, "bleu_stderr": 0.10033840689595877, "rouge1_fmeasure": 0.09784665332111649, "rouge1_fmeasure_stderr": 0.0016192055617561215, "rouge1_precision": 0.10821789424681753, "rouge1_precision_stderr": 0.002096638300657934, "rouge1_recall": 0.12472730510559357, "rouge1_recall_stderr": 0.002322606633118786, "rouge2_fmeasure": 0.010690797458122308, "rouge2_fmeasure_stderr": 0.0005103079982659685, "rouge2_precision": 0.01129846864454297, "rouge2_precision_stderr": 0.0006277146654822384, "rouge2_recall": 0.015276009798409365, "rouge2_recall_stderr": 0.0008518486109567291, "rougeL_fmeasure": 0.08028298986501597, "rougeL_fmeasure_stderr": 0.0012500164736239873, "rougeL_precision": 0.08908785284327644, "rougeL_precision_stderr": 0.001705722903459829, "rougeL_recall": 0.10389230433583506, "rougeL_recall_stderr": 0.0019161308146752112, "rougeLsum_fmeasure": 0.09248642582400225, "rougeLsum_fmeasure_stderr": 0.0015079304696231108, "rougeLsum_precision": 0.10264516065485063, "rougeLsum_precision_stderr": 0.0019926607627734193, "rougeLsum_recall": 0.11786995367940563, "rougeLsum_recall_stderr": 0.0021718471770820493}}, "e2e_nlg_cleaned": {"generate_text_restaurant": {"bleu": 4.732867354096951, "bleu_stderr": 0.15404543486723934, "rouge1_fmeasure": 0.30378389260311006, "rouge1_fmeasure_stderr": 0.0020385063291000773, "rouge1_precision": 0.35442414475500134, "rouge1_precision_stderr": 0.002731868155718904, "rouge1_recall": 0.29373594326712926, "rouge1_recall_stderr": 0.002294258375803093, "rouge2_fmeasure": 0.08882317107094179, "rouge2_fmeasure_stderr": 0.0014555231709394456, "rouge2_precision": 0.10512573374019807, "rouge2_precision_stderr": 0.0018530498330245938, "rouge2_recall": 0.08588813414618772, "rouge2_recall_stderr": 0.0014835043277026077, "rougeL_fmeasure": 0.2256061529203063, "rougeL_fmeasure_stderr": 0.0016287697579368406, "rougeL_precision": 0.2649706992736119, "rougeL_precision_stderr": 0.0022811580211275924, "rougeL_recall": 0.218297790406449, "rougeL_recall_stderr": 0.001833736502924904, "rougeLsum_fmeasure": 0.25364769619114425, "rougeLsum_fmeasure_stderr": 0.0018650391381006888, "rougeLsum_precision": 0.2972954010894401, "rougeLsum_precision_stderr": 0.002533132838252456, "rougeLsum_recall": 0.24506290855487275, "rougeLsum_recall_stderr": 0.002062388974631323}}, "gem_xsum": {"article_DOC_summary": {"bleu": 0.6437348489109824, "bleu_stderr": 0.08105590125094665, "rouge1_fmeasure": 0.11439654244933288, "rouge1_fmeasure_stderr": 0.0022963595160460466, "rouge1_precision": 0.08457756400468115, "rouge1_precision_stderr": 0.0018422703241089745, "rouge1_recall": 0.195876410222803, "rouge1_recall_stderr": 0.0038335449412391716, "rouge2_fmeasure": 0.0159331168872316, "rouge2_fmeasure_stderr": 0.0009893346680723053, "rouge2_precision": 0.011624998600318197, "rouge2_precision_stderr": 0.0007516865101337179, "rouge2_recall": 0.02749715301034778, "rouge2_recall_stderr": 0.0016328340556882883, "rougeL_fmeasure": 0.0973492647648112, "rougeL_fmeasure_stderr": 0.0017981023986070029, "rougeL_precision": 0.07194115142590224, "rougeL_precision_stderr": 0.0014722611772001854, "rougeL_recall": 0.167560514280581, "rougeL_recall_stderr": 0.003062295835710636, "rougeLsum_fmeasure": 0.09140331616988624, "rougeLsum_fmeasure_stderr": 0.0017813872559020623, "rougeLsum_precision": 0.06768573146684626, "rougeLsum_precision_stderr": 0.0014754231743952838, "rougeLsum_recall": 0.15747481060321722, "rougeLsum_recall_stderr": 0.0030260328022856386}}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/merged_4shots.csv b/1b11b51b5/evaluation/generation/merged_4shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..13d13927f5a69eea780d427520c6626093645e1b
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_4shots.csv
@@ -0,0 +1,5 @@
+dataset,prompt,metric,value
+e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.09490796525239133
+gem_xsum,article_DOC_summary,rouge2_fmeasure,0.00410550208994294
+web_nlg_en,PALM_prompt,rouge2_fmeasure,0.03236773058084414
+wiki_lingua_en,tldr_en,rouge2_fmeasure,0.004931876443326115
diff --git a/1b11b51b5/evaluation/generation/merged_4shots.json b/1b11b51b5/evaluation/generation/merged_4shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c7a6e2367a8cefe473db398fbafaa4fa58c1f51
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_4shots.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"PALM_prompt": {"bleu": 0.19732756628191997, "bleu_stderr": 0.026658287671472842, "rouge1_fmeasure": 0.08257826480436961, "rouge1_fmeasure_stderr": 0.0018872309614066823, "rouge1_precision": 0.06070458892233607, "rouge1_precision_stderr": 0.0017816786117538406, "rouge1_recall": 0.19093902196152424, "rouge1_recall_stderr": 0.003753006790856248, "rouge2_fmeasure": 0.03236773058084414, "rouge2_fmeasure_stderr": 0.001033048999274749, "rouge2_precision": 0.02314057822422759, "rouge2_precision_stderr": 0.0009303717509439983, "rouge2_recall": 0.08094570128568006, "rouge2_recall_stderr": 0.002426051231637763, "rougeL_fmeasure": 0.07788575137246734, "rougeL_fmeasure_stderr": 0.001720896690999757, "rougeL_precision": 0.05654666196831523, "rougeL_precision_stderr": 0.001579449111979041, "rougeL_recall": 0.18383917337722214, "rougeL_recall_stderr": 0.0036538299358506152, "rougeLsum_fmeasure": 0.07823625277495454, "rougeLsum_fmeasure_stderr": 0.0017358911503115172, "rougeLsum_precision": 0.057097033183754045, "rougeLsum_precision_stderr": 0.0016220022129662945, "rougeLsum_recall": 0.1838313736752301, "rougeLsum_recall_stderr": 0.003621428295981444}}, "GEM/wiki_lingua_en": {"tldr_en": {"bleu": 0.11289573429538445, "bleu_stderr": 0.01845710657006998, "rouge1_fmeasure": 0.03524490990351491, "rouge1_fmeasure_stderr": 0.0013594404865196162, "rouge1_precision": 0.04157996091862478, "rouge1_precision_stderr": 0.0018032030088400309, "rouge1_recall": 0.044926460912185065, "rouge1_recall_stderr": 0.0018528180948563747, "rouge2_fmeasure": 0.004931876443326115, "rouge2_fmeasure_stderr": 0.00038095104223179015, "rouge2_precision": 0.006050502360776094, "rouge2_precision_stderr": 0.0005836053795733668, "rouge2_recall": 0.007180836611458644, "rouge2_recall_stderr": 0.0006625116906510873, "rougeL_fmeasure": 0.029054409914181435, "rougeL_fmeasure_stderr": 0.001089301348325835, "rougeL_precision": 0.03456113363014217, "rougeL_precision_stderr": 0.001498234658219552, "rougeL_recall": 0.0375951446671367, "rougeL_recall_stderr": 0.0015455223646234392, "rougeLsum_fmeasure": 0.03263081498138409, "rougeLsum_fmeasure_stderr": 0.0012475593919046596, "rougeLsum_precision": 0.03887702645834208, "rougeLsum_precision_stderr": 0.001695261016406579, "rougeLsum_recall": 0.04140726881716476, "rougeLsum_recall_stderr": 0.0016967910317344356}}, "e2e_nlg_cleaned": {"generate_text_restaurant": {"bleu": 5.210131054318675, "bleu_stderr": 0.16155247866216102, "rouge1_fmeasure": 0.30893188850915426, "rouge1_fmeasure_stderr": 0.002078050469669259, "rouge1_precision": 0.36395994345002086, "rouge1_precision_stderr": 0.002850604700096735, "rouge1_recall": 0.29667153746092684, "rouge1_recall_stderr": 0.0022956768105348727, "rouge2_fmeasure": 0.09490796525239133, "rouge2_fmeasure_stderr": 0.0015347079760504483, "rouge2_precision": 0.1140043958983537, "rouge2_precision_stderr": 0.00202343696017089, "rouge2_recall": 0.09103549434582939, "rouge2_recall_stderr": 0.001530348953726892, "rougeL_fmeasure": 0.23092897642137852, "rougeL_fmeasure_stderr": 0.0016828405073314597, "rougeL_precision": 0.2740532547932312, "rougeL_precision_stderr": 0.0024194755484310826, "rougeL_recall": 0.2217823173210715, "rougeL_recall_stderr": 0.0018382921326583175, "rougeLsum_fmeasure": 0.2590295247906419, "rougeLsum_fmeasure_stderr": 0.0019213450001221285, "rougeLsum_precision": 0.30628692608188124, "rougeLsum_precision_stderr": 0.0026427962890604785, "rougeLsum_recall": 0.24850367361793804, "rougeLsum_recall_stderr": 0.002082869893324162}}, "gem_xsum": {"article_DOC_summary": {"bleu": 0.22579456084743735, "bleu_stderr": 0.0510480477202509, "rouge1_fmeasure": 0.03189372524424132, "rouge1_fmeasure_stderr": 0.0019572020109923736, "rouge1_precision": 0.026491777281042975, "rouge1_precision_stderr": 0.0018068087051168196, "rouge1_recall": 0.0494491413339096, "rouge1_recall_stderr": 0.003027387600352431, "rouge2_fmeasure": 0.00410550208994294, "rouge2_fmeasure_stderr": 0.00047252936288694365, "rouge2_precision": 0.0034382794036798173, "rouge2_precision_stderr": 0.00047491954770546336, "rouge2_recall": 0.006467065967569708, "rouge2_recall_stderr": 0.0007276044578978292, "rougeL_fmeasure": 0.027101703724227193, "rougeL_fmeasure_stderr": 0.0016162858861853424, "rougeL_precision": 0.02246733493087934, "rougeL_precision_stderr": 0.0015021222768415922, "rougeL_recall": 0.042219853518797514, "rougeL_recall_stderr": 0.0025299073135696837, "rougeLsum_fmeasure": 0.025143845266448906, "rougeLsum_fmeasure_stderr": 0.0015404507929384814, "rougeLsum_precision": 0.02108679945636943, "rougeLsum_precision_stderr": 0.0014787824836100922, "rougeLsum_recall": 0.03900351353665605, "rougeLsum_recall_stderr": 0.002373549357207589}}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/merged_5shots.csv b/1b11b51b5/evaluation/generation/merged_5shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1593c386565a7afc4f4861cbeffe0da007309b28
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_5shots.csv
@@ -0,0 +1,5 @@
+dataset,prompt,metric,value
+e2e_nlg_cleaned,generate_text_restaurant,rouge2_fmeasure,0.09841408639827368
+gem_xsum,article_DOC_summary,rouge2_fmeasure,0.0001313177797301129
+web_nlg_en,PALM_prompt,rouge2_fmeasure,0.0300231413049405
+wiki_lingua_en,tldr_en,rouge2_fmeasure,0.0009425275750514962
diff --git a/1b11b51b5/evaluation/generation/merged_5shots.json b/1b11b51b5/evaluation/generation/merged_5shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..9658f4e90a7b47358a514f54512e172472a4e6d1
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/merged_5shots.json
@@ -0,0 +1 @@
+{"GEM/web_nlg_en": {"PALM_prompt": {"bleu": 0.14367082750773918, "bleu_stderr": 0.01727398003487744, "rouge1_fmeasure": 0.08023946961419831, "rouge1_fmeasure_stderr": 0.0018599842014871484, "rouge1_precision": 0.061364801134236466, "rouge1_precision_stderr": 0.0019895158236849513, "rouge1_recall": 0.1852053079126511, "rouge1_recall_stderr": 0.0037095859419792, "rouge2_fmeasure": 0.0300231413049405, "rouge2_fmeasure_stderr": 0.0009672037164958253, "rouge2_precision": 0.02155110915099608, "rouge2_precision_stderr": 0.000944250904374556, "rouge2_recall": 0.0760367527800932, "rouge2_recall_stderr": 0.0023514862121833038, "rougeL_fmeasure": 0.07552378160245939, "rougeL_fmeasure_stderr": 0.0016866442023514398, "rougeL_precision": 0.056807969707911, "rougeL_precision_stderr": 0.0017323813715249387, "rougeL_recall": 0.17835942307556787, "rougeL_recall_stderr": 0.003607251708504229, "rougeLsum_fmeasure": 0.0756658622438174, "rougeLsum_fmeasure_stderr": 0.0017015828692780743, "rougeLsum_precision": 0.05708185625481882, "rougeLsum_precision_stderr": 0.0017532405619586147, "rougeLsum_recall": 0.1779104329111164, "rougeLsum_recall_stderr": 0.0035688476761420685}}, "GEM/wiki_lingua_en": {"tldr_en": {"bleu": 2.1707923159701563e-10, "bleu_stderr": 1.2012188612881837e-09, "rouge1_fmeasure": 0.006160737725223056, "rouge1_fmeasure_stderr": 0.0006512100088223974, "rouge1_precision": 0.008254948637118429, "rouge1_precision_stderr": 0.0009459136809570537, "rouge1_recall": 0.007562240110686054, "rouge1_recall_stderr": 0.0008636106012690946, "rouge2_fmeasure": 0.0009425275750514962, "rouge2_fmeasure_stderr": 0.00018272575729318913, "rouge2_precision": 0.0012474030450093298, "rouge2_precision_stderr": 0.00025491125834316595, "rouge2_recall": 0.0013793630128949251, "rouge2_recall_stderr": 0.00032763950112405063, "rougeL_fmeasure": 0.00501321248753611, "rougeL_fmeasure_stderr": 0.0005224006471877824, "rougeL_precision": 0.0067843630893508035, "rougeL_precision_stderr": 0.0007756800649250622, "rougeL_recall": 0.006224422456967982, "rougeL_recall_stderr": 0.0007102549709126986, "rougeLsum_fmeasure": 0.005755424216995409, "rougeLsum_fmeasure_stderr": 0.0006110349341052841, "rougeLsum_precision": 0.007737895530410426, "rougeLsum_precision_stderr": 0.0008942014082809357, "rougeLsum_recall": 0.006995766927183111, "rougeLsum_recall_stderr": 0.0007965814705828734}}, "e2e_nlg_cleaned": {"generate_text_restaurant": {"bleu": 5.3388680014012015, "bleu_stderr": 0.13008777046551853, "rouge1_fmeasure": 0.3120715598551173, "rouge1_fmeasure_stderr": 0.002049220094979265, "rouge1_precision": 0.36808074646470906, "rouge1_precision_stderr": 0.0028756888288344754, "rouge1_recall": 0.2997544520993915, "rouge1_recall_stderr": 0.0022774797385096417, "rouge2_fmeasure": 0.09841408639827368, "rouge2_fmeasure_stderr": 0.0015376685877858227, "rouge2_precision": 0.11888183945969535, "rouge2_precision_stderr": 0.0020436399790904566, "rouge2_recall": 0.09438735759794968, "rouge2_recall_stderr": 0.00155057405248203, "rougeL_fmeasure": 0.2360616277086933, "rougeL_fmeasure_stderr": 0.0016771641427817245, "rougeL_precision": 0.2806218677955488, "rougeL_precision_stderr": 0.002465381284480861, "rougeL_recall": 0.22659913710533197, "rougeL_recall_stderr": 0.0018404855639564178, "rougeLsum_fmeasure": 0.26331410062448574, "rougeLsum_fmeasure_stderr": 0.0018958344026201047, "rougeLsum_precision": 0.3119441169782797, "rougeLsum_precision_stderr": 0.0026747982066311175, "rougeLsum_recall": 0.2526371639744921, "rougeLsum_recall_stderr": 0.0020647828708917995}}, "gem_xsum": {"article_DOC_summary": {"bleu": 1.5544568735399087e-43, "bleu_stderr": 1.081029400958023e-37, "rouge1_fmeasure": 0.0024394449960508753, "rouge1_fmeasure_stderr": 0.000649872781418437, "rouge1_precision": 0.0029325830197336608, "rouge1_precision_stderr": 0.0007845716748468843, "rouge1_recall": 0.002212272201507509, "rouge1_recall_stderr": 0.0006115672097018799, "rouge2_fmeasure": 0.0001313177797301129, "rouge2_fmeasure_stderr": 9.625094432461928e-05, "rouge2_precision": 0.00017152658662092626, "rouge2_precision_stderr": 0.00012780446523672458, "rouge2_recall": 0.00010681142756614455, "rouge2_recall_stderr": 7.755982479808459e-05, "rougeL_fmeasure": 0.0019268220007723433, "rougeL_fmeasure_stderr": 0.0005091814465631191, "rougeL_precision": 0.0023349726205139387, "rougeL_precision_stderr": 0.0006284522046757939, "rougeL_recall": 0.0017335948670290622, "rougeL_recall_stderr": 0.0004670903743357543, "rougeLsum_fmeasure": 0.002087647218353957, "rougeLsum_fmeasure_stderr": 0.0005649060898529495, "rougeLsum_precision": 0.00249935226602566, "rougeLsum_precision_stderr": 0.0006799933367648228, "rougeLsum_recall": 0.0019134028298558926, "rougeLsum_recall_stderr": 0.0005458175188244193}}}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..518561f2ba296b614dca7c7f5fa69f713504410d
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.1715109274753528,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.02642539468126329
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.056271792717741465,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0020834882225830133
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.17970657292444336,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037758846666814077
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07392470782815987,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.00177901763739792
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.021453121863695197,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0010399844752529916
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07781374849374786,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.002432084660851988
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.03028468748562561,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0010296918553042514
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.054104790772534245,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.002018866888899467
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17534202443784297,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.00369978730480511
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07124999476464311,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016868036669108117
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.054126123805613165,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002026505118360271
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1747286304108634,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003660334571889786
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07115192816057488,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016843849706888723
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..5beb040990b91045ed2b21605a98e3d8e52d79c0
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.11694864689561944,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.010760323698374666
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.05492023407017857,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0016326725952034475
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.17990254142984613,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037492925899754617
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07558454105004475,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017728413797174199
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.02075424080508391,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009150263161295265
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.0760260739680381,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0023774787682776578
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.02956013283155878,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009596682875711376
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05153123991260807,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0014581185184364613
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17447931108768522,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0036741060344485967
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07194166497021176,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001647469505361452
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.052002469160781814,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015071594845800093
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.17445084715749506,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0036515775079597516
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07220041477281516,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016608929228483667
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..0f8bb06ba05cc00890edfe100ab8916e045ade36
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.157083884873114,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.025447861405185956
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.059290722393941356,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019300973301455053
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.18222315758965466,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037052853928907043
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.07793163760494656,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0017891627195254817
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.021478064415916574,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009791043575277162
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.07687939137894265,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0023619853957239805
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.029975159373482684,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009737894387487846
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05528292115358125,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0017712704886265283
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17551310076734894,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0036271666172633756
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07339358383221016,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016455203554569008
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.05583469963213412,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0018018226490151468
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.17544849623130826,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0035888045371675796
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07381888878270101,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0016588962455865598
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..2cb8bb1708c83e819e31cc0b24f665ee674ac711
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.19732756628191997,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.026658287671472842
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.06070458892233607,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017816786117538406
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.19093902196152424,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.003753006790856248
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.08257826480436961,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018872309614066823
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.02314057822422759,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0009303717509439983
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.08094570128568006,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.002426051231637763
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.03236773058084414,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001033048999274749
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.05654666196831523,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001579449111979041
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.18383917337722214,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0036538299358506152
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07788575137246734,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001720896690999757
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.057097033183754045,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016220022129662945
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1838313736752301,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.003621428295981444
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.07823625277495454,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017358911503115172
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json
new file mode 100644
index 0000000000000000000000000000000000000000..8bf674b3e2ef1c5453c8d400b6c8b7b956f7d161
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-web_nlg_en.templates=PALM_prompt.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:35.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "bleu": 0.14367082750773918,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.01727398003487744
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_precision": 0.061364801134236466,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0019895158236849513
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_recall": 0.1852053079126511,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0037095859419792
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge1_fmeasure": 0.08023946961419831,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0018599842014871484
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_precision": 0.02155110915099608,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.000944250904374556
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_recall": 0.0760367527800932,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0023514862121833038
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rouge2_fmeasure": 0.0300231413049405,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0009672037164958253
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_precision": 0.056807969707911,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0017323813715249387
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_recall": 0.17835942307556787,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.003607251708504229
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeL_fmeasure": 0.07552378160245939,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016866442023514398
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_precision": 0.05708185625481882,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0017532405619586147
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_recall": 0.1779104329111164,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0035688476761420685
+    },
+    {
+      "task_name": "GEM/web_nlg_en",
+      "prompt_name": "PALM_prompt",
+      "rougeLsum_fmeasure": 0.0756658622438174,
+      "dataset_path": "GEM/web_nlg",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0017015828692780743
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..7eae5ab20764cb86674480419b176aea0a19fbe5
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.11243090334580715,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0015900812709383934
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.11715651177590802,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.001696871084875322
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.10058251758979377,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0012559808866013141
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.006128533869165112,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0004018707853940709
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.007425968853967204,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0005962348292070988
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.005682283734931176,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00035696819267689325
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.09041052243361407,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0012316877506636582
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.09424789347821856,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0013396373126375609
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08035070126772184,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0009302451261276216
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.10789218625357785,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0015108158731484223
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.11248330566212135,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016035045599026544
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.09651169925228414,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0011859501014305082
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.4868015971525538,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.08360711580226779
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..97ae12368c882e2c693cdb152949bc40d3e4597d
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.11992316681250555,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0017675704654899571
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.13948810001589107,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002058649220725351
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.11194017044626671,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0014428492038973415
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.010522981314992496,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0006069518989281641
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.014385796237686857,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0008041815234389601
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.010243977830737588,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.000493265280559601
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.09634769905116539,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0013910007655887457
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.11364453759174203,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0016847934987669815
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.0897361308600908,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0010822617010161773
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.11382504495028803,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0016623075783859457
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.1321927492875554,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0019214290897240306
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.10607719694851259,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0013391525579239643
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.7583577300885451,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.0594734221378448
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0e5233baebf9c686f69d320cd28f107fc73671e
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.10821789424681753,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.002096638300657934
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.12472730510559357,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.002322606633118786
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.09784665332111649,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0016192055617561215
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.01129846864454297,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0006277146654822384
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.015276009798409365,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0008518486109567291
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.010690797458122308,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0005103079982659685
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.08908785284327644,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001705722903459829
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.10389230433583506,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0019161308146752112
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.08028298986501597,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0012500164736239873
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.10264516065485063,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0019926607627734193
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.11786995367940563,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0021718471770820493
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.09248642582400225,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0015079304696231108
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.8537954880656907,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.10033840689595877
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..1289e6467ce8d7cbaa03fec395f6a26a289451f1
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.04157996091862478,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0018032030088400309
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.044926460912185065,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0018528180948563747
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.03524490990351491,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0013594404865196162
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.006050502360776094,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.0005836053795733668
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.007180836611458644,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.0006625116906510873
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.004931876443326115,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00038095104223179015
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.03456113363014217,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.001498234658219552
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.0375951446671367,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0015455223646234392
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.029054409914181435,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.001089301348325835
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.03887702645834208,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.001695261016406579
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.04140726881716476,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0016967910317344356
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.03263081498138409,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0012475593919046596
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 0.11289573429538445,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 0.01845710657006998
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..fe688f4ce0b791ece62bd8ddbfce31567488467d
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=GEM-wiki_lingua_en.templates=tldr_en.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_precision": 0.008254948637118429,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_precision_stderr": 0.0009459136809570537
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_recall": 0.007562240110686054,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_recall_stderr": 0.0008636106012690946
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge1_fmeasure": 0.006160737725223056,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0006512100088223974
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_precision": 0.0012474030450093298,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_precision_stderr": 0.00025491125834316595
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_recall": 0.0013793630128949251,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_recall_stderr": 0.00032763950112405063
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rouge2_fmeasure": 0.0009425275750514962,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.00018272575729318913
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_precision": 0.0067843630893508035,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_precision_stderr": 0.0007756800649250622
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_recall": 0.006224422456967982,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_recall_stderr": 0.0007102549709126986
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeL_fmeasure": 0.00501321248753611,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0005224006471877824
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_precision": 0.007737895530410426,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0008942014082809357
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_recall": 0.006995766927183111,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0007965814705828734
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "rougeLsum_fmeasure": 0.005755424216995409,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0006110349341052841
+    },
+    {
+      "task_name": "GEM/wiki_lingua_en",
+      "prompt_name": "tldr_en",
+      "bleu": 2.1707923159701563e-10,
+      "dataset_path": "GEM/wiki_lingua",
+      "dataset_name": "en",
+      "subset": null,
+      "bleu_stderr": 1.2012188612881837e-09
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..99bdd804edbe591661307fbd36a088d56a700f5e
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 3.741207237448518,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.12328594100727748
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.33565838168296036,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002543739891022053
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.27743476294626046,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022564269895565987
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.28858251510563476,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.001991339721709451
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.08431013375181877,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0015876617488378608
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.07011453690760112,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001335315815509775
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.07273933980217488,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0013272082416632288
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.24464584594762856,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0020076748803741955
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.20102123374490308,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017273862409986716
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.20937991809553785,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015406451796370017
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.2774307167772055,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0023056419978161836
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.22843062477167406,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.001987724352272349
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.23796164791282326,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.001803514609873287
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..b313ae5e772d2a5c41dd7656a0674f4c00a34b40
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 4.290890219148563,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.11797671966393032
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.3459713718406746,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002718482058174072
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.28815608021642963,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022672787761088384
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.2978425731750224,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020372027980421125
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.09498590594242383,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0017360545885379483
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.07791865939566535,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014093737049867404
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.08070496923908829,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.001400104340028372
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2542525441295202,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0021937081691301894
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.2106927728428706,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0017758508939911513
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.21768665558633662,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0015933877730589948
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.2862920446603683,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002475870560064218
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.2376715264121337,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002037562231835293
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.24573333528025115,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018602140974894146
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..ee74ee3b84972807184a01f4c3424a5895ffab75
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 4.732867354096951,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.15404543486723934
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.35442414475500134,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002731868155718904
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.29373594326712926,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.002294258375803093
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.30378389260311006,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.0020385063291000773
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.10512573374019807,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0018530498330245938
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.08588813414618772,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.0014835043277026077
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.08882317107094179,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0014555231709394456
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2649706992736119,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0022811580211275924
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.218297790406449,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.001833736502924904
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.2256061529203063,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016287697579368406
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.2972954010894401,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.002533132838252456
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.24506290855487275,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002062388974631323
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.25364769619114425,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018650391381006888
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..6d234fb14d38e170dfb0ee20a4c8859cbd129231
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 5.210131054318675,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.16155247866216102
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.36395994345002086,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.002850604700096735
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.29667153746092684,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022956768105348727
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.30893188850915426,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002078050469669259
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.1140043958983537,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.00202343696017089
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.09103549434582939,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.001530348953726892
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.09490796525239133,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015347079760504483
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2740532547932312,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.0024194755484310826
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.2217823173210715,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018382921326583175
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.23092897642137852,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016828405073314597
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.30628692608188124,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026427962890604785
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.24850367361793804,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.002082869893324162
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.2590295247906419,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0019213450001221285
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..cc1ed0693cd0fba620ffd2661d8e8ae1b2bf9299
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=e2e_nlg_cleaned.templates=generate_text_restaurant.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "bleu": 5.3388680014012015,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "bleu_stderr": 0.13008777046551853
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_precision": 0.36808074646470906,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_precision_stderr": 0.0028756888288344754
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_recall": 0.2997544520993915,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_recall_stderr": 0.0022774797385096417
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge1_fmeasure": 0.3120715598551173,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge1_fmeasure_stderr": 0.002049220094979265
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_precision": 0.11888183945969535,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_precision_stderr": 0.0020436399790904566
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_recall": 0.09438735759794968,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_recall_stderr": 0.00155057405248203
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rouge2_fmeasure": 0.09841408639827368,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rouge2_fmeasure_stderr": 0.0015376685877858227
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_precision": 0.2806218677955488,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_precision_stderr": 0.002465381284480861
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_recall": 0.22659913710533197,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_recall_stderr": 0.0018404855639564178
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeL_fmeasure": 0.2360616277086933,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeL_fmeasure_stderr": 0.0016771641427817245
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_precision": 0.3119441169782797,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_precision_stderr": 0.0026747982066311175
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_recall": 0.2526371639744921,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_recall_stderr": 0.0020647828708917995
+    },
+    {
+      "task_name": "e2e_nlg_cleaned",
+      "prompt_name": "generate_text_restaurant",
+      "rougeLsum_fmeasure": 0.26331410062448574,
+      "dataset_path": "e2e_nlg_cleaned",
+      "dataset_name": null,
+      "subset": null,
+      "rougeLsum_fmeasure_stderr": 0.0018958344026201047
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
new file mode 100644
index 0000000000000000000000000000000000000000..073ce4f13b7cb9a7dd529612724067ad800d66a2
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=1.batchsize=16.seed=1234.timestamp=2023-01-22T19:19:06.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.09734130580010827,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.001771813879632219
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.23205596880640392,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003968640338152418
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.1347806025835107,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002364452110807362
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.013809019824869628,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007842126128288541
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.03384878468635188,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0018246616684185503
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.01937279969544618,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0010767896556060932
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07877507926978208,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.001317195018620984
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.18918199375900235,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0029962553876845713
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.10920408732481979,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017466454015791766
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.07655704256204823,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014263611561085394
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.18366939517842565,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0032532823145414863
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.10609898226051803,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.001907016281645563
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.7556443558763775,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.08356354268542657
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 1,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..ad469e1c6e82052c83e70c963ceb35ec4abc7443
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=2.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.08876555984023238,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0016308308833255663
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.21454525270713967,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0035861819885292467
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.12357434836180202,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.002149017836427172
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.011772595475451241,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007152713643140575
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.02914764125397616,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.001651586800843389
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.01649554595160008,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.000971551127185768
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07425164547734261,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0012476815962285165
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.18105447617675274,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.002824838892161731
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.10362842140926162,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016508490182790031
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.06960818379765893,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0013153158283565624
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.17002840065420005,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002984809197834264
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.09716065684449275,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0017475396647593806
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.5986931718476021,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.08317206014621986
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 2,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
new file mode 100644
index 0000000000000000000000000000000000000000..17c2c006cdf250299549025161d2612d558f980a
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=3.batchsize=16.seed=1234.timestamp=2023-01-24T16:59:52.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.08457756400468115,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0018422703241089745
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.195876410222803,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0038335449412391716
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.11439654244933288,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0022963595160460466
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.011624998600318197,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.0007516865101337179
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.02749715301034778,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0016328340556882883
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.0159331168872316,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.0009893346680723053
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.07194115142590224,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0014722611772001854
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.167560514280581,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.003062295835710636
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.0973492647648112,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0017981023986070029
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.06768573146684626,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014754231743952838
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.15747481060321722,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0030260328022856386
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.09140331616988624,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0017813872559020623
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.6437348489109824,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.08105590125094665
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 3,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json
new file mode 100644
index 0000000000000000000000000000000000000000..f68b2a2b41e566dee0977d7c0af6a8745b8ad967
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=4.batchsize=16.seed=1234.timestamp=2023-01-24T15:49:56.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.026491777281042975,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0018068087051168196
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.0494491413339096,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.003027387600352431
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.03189372524424132,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.0019572020109923736
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.0034382794036798173,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00047491954770546336
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.006467065967569708,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 0.0007276044578978292
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.00410550208994294,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 0.00047252936288694365
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.02246733493087934,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0015021222768415922
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.042219853518797514,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0025299073135696837
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.027101703724227193,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0016162858861853424
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.02108679945636943,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0014787824836100922
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.03900351353665605,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.002373549357207589
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.025143845266448906,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0015404507929384814
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 0.22579456084743735,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 0.0510480477202509
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 4,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
new file mode 100644
index 0000000000000000000000000000000000000000..c1ddb35b4d2e8772a0a97321fe87c6db85a225ee
--- /dev/null
+++ b/1b11b51b5/evaluation/generation/slim.limited=3000.model=lm1-1b1-1b5.task=gem_xsum.templates=article_DOC_summary.fewshot=5.batchsize=16.seed=1234.timestamp=2023-01-24T15:52:02.json
@@ -0,0 +1,133 @@
+{
+  "results": [
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_precision": 0.0029325830197336608,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_precision_stderr": 0.0007845716748468843
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_recall": 0.002212272201507509,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_recall_stderr": 0.0006115672097018799
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge1_fmeasure": 0.0024394449960508753,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge1_fmeasure_stderr": 0.000649872781418437
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_precision": 0.00017152658662092626,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_precision_stderr": 0.00012780446523672458
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_recall": 0.00010681142756614455,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_recall_stderr": 7.755982479808459e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rouge2_fmeasure": 0.0001313177797301129,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rouge2_fmeasure_stderr": 9.625094432461928e-05
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_precision": 0.0023349726205139387,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_precision_stderr": 0.0006284522046757939
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_recall": 0.0017335948670290622,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_recall_stderr": 0.0004670903743357543
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeL_fmeasure": 0.0019268220007723433,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeL_fmeasure_stderr": 0.0005091814465631191
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_precision": 0.00249935226602566,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_precision_stderr": 0.0006799933367648228
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_recall": 0.0019134028298558926,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_recall_stderr": 0.0005458175188244193
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "rougeLsum_fmeasure": 0.002087647218353957,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "rougeLsum_fmeasure_stderr": 0.0005649060898529495
+    },
+    {
+      "task_name": "gem_xsum",
+      "prompt_name": "article_DOC_summary",
+      "bleu": 1.5544568735399087e-43,
+      "dataset_path": "GEM/xsum",
+      "dataset_name": null,
+      "subset": "",
+      "bleu_stderr": 1.081029400958023e-37
+    }
+  ],
+  "config": {
+    "model": "hf-causal",
+    "model_args": "pretrained=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-1b1-1b5/transformers,use_accelerate=True,tokenizer=/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/gpt2,dtype=bfloat16",
+    "task_args": "",
+    "num_fewshot": 5,
+    "batch_size": 16,
+    "device": "cuda",
+    "use_cache": false,
+    "limit": 3000,
+    "bootstrap_iters": 10,
+    "seed": 1234
+  }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv b/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..fad151891ae1a9273eff606e9b860b2d0a215d40
--- /dev/null
+++ b/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.318,0.014734079309311901,0
+anli_r2,acc,0.338,0.014965960710224473,0
+anli_r3,acc,0.3275,0.01355321116725195,0
+arc_challenge,acc,0.1621160409556314,0.01077026738079835,0
+arc_challenge,acc_norm,0.2090443686006826,0.011882746987406446,0
+arc_easy,acc,0.3303872053872054,0.009651430216428194,0
+arc_easy,acc_norm,0.3253367003367003,0.00961342770899619,0
+boolq,acc,0.5850152905198777,0.008617716361921567,1
+cb,acc,0.48214285714285715,0.06737697508644648,1
+cb,f1,0.33636151875145587,,1
+copa,acc,0.55,0.04999999999999999,0
+hellaswag,acc,0.26379207329217286,0.00439787247185492,0
+hellaswag,acc_norm,0.2671778530173272,0.004415816696303073,0
+piqa,acc,0.573993471164309,0.011537375448519445,0
+piqa,acc_norm,0.5652883569096845,0.011565943814308855,0
+rte,acc,0.49097472924187724,0.030091559826331334,0
+sciq,acc,0.547,0.015749255189977586,0
+sciq,acc_norm,0.534,0.015782683329937614,0
+storycloze_2016,acc,0.5189738107963656,0.011554104174019695,0
+winogrande,acc,0.5146014206787688,0.014046492383275837,0
diff --git a/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json b/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..d92a50a6e5ef7ddd92576811010c813f2fd96918
--- /dev/null
+++ b/1b11b51b5/evaluation/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.318,
+            "acc_stderr": 0.014734079309311901
+        },
+        "anli_r2": {
+            "acc": 0.338,
+            "acc_stderr": 0.014965960710224473
+        },
+        "anli_r3": {
+            "acc": 0.3275,
+            "acc_stderr": 0.01355321116725195
+        },
+        "cb": {
+            "acc": 0.48214285714285715,
+            "acc_stderr": 0.06737697508644648,
+            "f1": 0.33636151875145587
+        },
+        "copa": {
+            "acc": 0.55,
+            "acc_stderr": 0.04999999999999999
+        },
+        "hellaswag": {
+            "acc": 0.26379207329217286,
+            "acc_stderr": 0.00439787247185492,
+            "acc_norm": 0.2671778530173272,
+            "acc_norm_stderr": 0.004415816696303073
+        },
+        "rte": {
+            "acc": 0.49097472924187724,
+            "acc_stderr": 0.030091559826331334
+        },
+        "winogrande": {
+            "acc": 0.5146014206787688,
+            "acc_stderr": 0.014046492383275837
+        },
+        "storycloze_2016": {
+            "acc": 0.5189738107963656,
+            "acc_stderr": 0.011554104174019695
+        },
+        "boolq": {
+            "acc": 0.5850152905198777,
+            "acc_stderr": 0.008617716361921567
+        },
+        "arc_easy": {
+            "acc": 0.3303872053872054,
+            "acc_stderr": 0.009651430216428194,
+            "acc_norm": 0.3253367003367003,
+            "acc_norm_stderr": 0.00961342770899619
+        },
+        "arc_challenge": {
+            "acc": 0.1621160409556314,
+            "acc_stderr": 0.01077026738079835,
+            "acc_norm": 0.2090443686006826,
+            "acc_norm_stderr": 0.011882746987406446
+        },
+        "sciq": {
+            "acc": 0.547,
+            "acc_stderr": 0.015749255189977586,
+            "acc_norm": 0.534,
+            "acc_norm_stderr": 0.015782683329937614
+        },
+        "piqa": {
+            "acc": 0.573993471164309,
+            "acc_stderr": 0.011537375448519445,
+            "acc_norm": 0.5652883569096845,
+            "acc_norm_stderr": 0.011565943814308855
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..45306ef93f063dc28433c49346cbfd14f047bfb3
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.332,0.01489959724281149,0
+anli_r2,acc,0.308,0.01460648312734276,0
+anli_r3,acc,0.3233333333333333,0.013508372867300228,0
+arc_challenge,acc,0.16638225255972697,0.010883248065964142,0
+arc_challenge,acc_norm,0.21245733788395904,0.011953482906582952,0
+arc_easy,acc,0.3421717171717172,0.00973523677195875,0
+arc_easy,acc_norm,0.31186868686868685,0.009505823345817666,0
+boolq,acc,0.5529051987767584,0.00869596306417273,1
+cb,acc,0.4107142857142857,0.0663363415035954,1
+cb,f1,0.22313546423135466,,1
+copa,acc,0.54,0.05009082659620332,0
+hellaswag,acc,0.26269667396932883,0.004391995637542121,0
+hellaswag,acc_norm,0.264389563831906,0.004401063265803207,0
+piqa,acc,0.5788900979325353,0.01151970105915149,0
+piqa,acc_norm,0.5690968443960827,0.011553893175901412,0
+rte,acc,0.5342960288808665,0.030025579819366422,0
+sciq,acc,0.548,0.01574623586588068,0
+sciq,acc_norm,0.515,0.01581217964181491,0
+storycloze_2016,acc,0.5307322287546766,0.011540570846495544,0
+winogrande,acc,0.4980268350434096,0.014052376259225636,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..cfe3d4f9f9a028a1ca256c45bbbb7229f6157007
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-22-18-55-55_0shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.332,
+            "acc_stderr": 0.01489959724281149
+        },
+        "anli_r2": {
+            "acc": 0.308,
+            "acc_stderr": 0.01460648312734276
+        },
+        "anli_r3": {
+            "acc": 0.3233333333333333,
+            "acc_stderr": 0.013508372867300228
+        },
+        "cb": {
+            "acc": 0.4107142857142857,
+            "acc_stderr": 0.0663363415035954,
+            "f1": 0.22313546423135466
+        },
+        "copa": {
+            "acc": 0.54,
+            "acc_stderr": 0.05009082659620332
+        },
+        "hellaswag": {
+            "acc": 0.26269667396932883,
+            "acc_stderr": 0.004391995637542121,
+            "acc_norm": 0.264389563831906,
+            "acc_norm_stderr": 0.004401063265803207
+        },
+        "rte": {
+            "acc": 0.5342960288808665,
+            "acc_stderr": 0.030025579819366422
+        },
+        "winogrande": {
+            "acc": 0.4980268350434096,
+            "acc_stderr": 0.014052376259225636
+        },
+        "storycloze_2016": {
+            "acc": 0.5307322287546766,
+            "acc_stderr": 0.011540570846495544
+        },
+        "boolq": {
+            "acc": 0.5529051987767584,
+            "acc_stderr": 0.00869596306417273
+        },
+        "arc_easy": {
+            "acc": 0.3421717171717172,
+            "acc_stderr": 0.00973523677195875,
+            "acc_norm": 0.31186868686868685,
+            "acc_norm_stderr": 0.009505823345817666
+        },
+        "arc_challenge": {
+            "acc": 0.16638225255972697,
+            "acc_stderr": 0.010883248065964142,
+            "acc_norm": 0.21245733788395904,
+            "acc_norm_stderr": 0.011953482906582952
+        },
+        "sciq": {
+            "acc": 0.548,
+            "acc_stderr": 0.01574623586588068,
+            "acc_norm": 0.515,
+            "acc_norm_stderr": 0.01581217964181491
+        },
+        "piqa": {
+            "acc": 0.5788900979325353,
+            "acc_stderr": 0.01151970105915149,
+            "acc_norm": 0.5690968443960827,
+            "acc_norm_stderr": 0.011553893175901412
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..b43dc854f7bdcceb124f9437ceb78bc7a4093c19
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.323,0.014794927843348637,0
+anli_r2,acc,0.319,0.01474640486547348,0
+anli_r3,acc,0.3358333333333333,0.013639261190932884,0
+arc_challenge,acc,0.17235494880546076,0.011037113093461295,0
+arc_challenge,acc_norm,0.2098976109215017,0.011900548748047446,0
+arc_easy,acc,0.34764309764309764,0.009771868846830912,0
+arc_easy,acc_norm,0.3186026936026936,0.009560775507673364,0
+boolq,acc,0.5226299694189602,0.008736093428015826,1
+cb,acc,0.48214285714285715,0.06737697508644648,1
+cb,f1,0.3421052631578947,,1
+copa,acc,0.54,0.05009082659620332,0
+hellaswag,acc,0.2638916550487951,0.004398404992933851,0
+hellaswag,acc_norm,0.26419040031866164,0.004400000822742047,0
+piqa,acc,0.5723612622415669,0.011543009623282828,0
+piqa,acc_norm,0.5652883569096845,0.011565943814308855,0
+rte,acc,0.5306859205776173,0.03003973059219781,0
+sciq,acc,0.568,0.01567232023733621,0
+sciq,acc_norm,0.546,0.015752210388771844,0
+storycloze_2016,acc,0.5189738107963656,0.011554104174019694,0
+winogrande,acc,0.5090765588003157,0.014050170094497704,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..3fddcbbcad2bd9bd73785db6edf82c75c73a840d
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-23-19-58-55_1shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.323,
+            "acc_stderr": 0.014794927843348637
+        },
+        "anli_r2": {
+            "acc": 0.319,
+            "acc_stderr": 0.01474640486547348
+        },
+        "anli_r3": {
+            "acc": 0.3358333333333333,
+            "acc_stderr": 0.013639261190932884
+        },
+        "cb": {
+            "acc": 0.48214285714285715,
+            "acc_stderr": 0.06737697508644648,
+            "f1": 0.3421052631578947
+        },
+        "copa": {
+            "acc": 0.54,
+            "acc_stderr": 0.05009082659620332
+        },
+        "hellaswag": {
+            "acc": 0.2638916550487951,
+            "acc_stderr": 0.004398404992933851,
+            "acc_norm": 0.26419040031866164,
+            "acc_norm_stderr": 0.004400000822742047
+        },
+        "rte": {
+            "acc": 0.5306859205776173,
+            "acc_stderr": 0.03003973059219781
+        },
+        "winogrande": {
+            "acc": 0.5090765588003157,
+            "acc_stderr": 0.014050170094497704
+        },
+        "storycloze_2016": {
+            "acc": 0.5189738107963656,
+            "acc_stderr": 0.011554104174019694
+        },
+        "boolq": {
+            "acc": 0.5226299694189602,
+            "acc_stderr": 0.008736093428015826
+        },
+        "arc_easy": {
+            "acc": 0.34764309764309764,
+            "acc_stderr": 0.009771868846830912,
+            "acc_norm": 0.3186026936026936,
+            "acc_norm_stderr": 0.009560775507673364
+        },
+        "arc_challenge": {
+            "acc": 0.17235494880546076,
+            "acc_stderr": 0.011037113093461295,
+            "acc_norm": 0.2098976109215017,
+            "acc_norm_stderr": 0.011900548748047446
+        },
+        "sciq": {
+            "acc": 0.568,
+            "acc_stderr": 0.01567232023733621,
+            "acc_norm": 0.546,
+            "acc_norm_stderr": 0.015752210388771844
+        },
+        "piqa": {
+            "acc": 0.5723612622415669,
+            "acc_stderr": 0.011543009623282828,
+            "acc_norm": 0.5652883569096845,
+            "acc_norm_stderr": 0.011565943814308855
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..fad151891ae1a9273eff606e9b860b2d0a215d40
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.318,0.014734079309311901,0
+anli_r2,acc,0.338,0.014965960710224473,0
+anli_r3,acc,0.3275,0.01355321116725195,0
+arc_challenge,acc,0.1621160409556314,0.01077026738079835,0
+arc_challenge,acc_norm,0.2090443686006826,0.011882746987406446,0
+arc_easy,acc,0.3303872053872054,0.009651430216428194,0
+arc_easy,acc_norm,0.3253367003367003,0.00961342770899619,0
+boolq,acc,0.5850152905198777,0.008617716361921567,1
+cb,acc,0.48214285714285715,0.06737697508644648,1
+cb,f1,0.33636151875145587,,1
+copa,acc,0.55,0.04999999999999999,0
+hellaswag,acc,0.26379207329217286,0.00439787247185492,0
+hellaswag,acc_norm,0.2671778530173272,0.004415816696303073,0
+piqa,acc,0.573993471164309,0.011537375448519445,0
+piqa,acc_norm,0.5652883569096845,0.011565943814308855,0
+rte,acc,0.49097472924187724,0.030091559826331334,0
+sciq,acc,0.547,0.015749255189977586,0
+sciq,acc_norm,0.534,0.015782683329937614,0
+storycloze_2016,acc,0.5189738107963656,0.011554104174019695,0
+winogrande,acc,0.5146014206787688,0.014046492383275837,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..d92a50a6e5ef7ddd92576811010c813f2fd96918
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-53-33_2shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.318,
+            "acc_stderr": 0.014734079309311901
+        },
+        "anli_r2": {
+            "acc": 0.338,
+            "acc_stderr": 0.014965960710224473
+        },
+        "anli_r3": {
+            "acc": 0.3275,
+            "acc_stderr": 0.01355321116725195
+        },
+        "cb": {
+            "acc": 0.48214285714285715,
+            "acc_stderr": 0.06737697508644648,
+            "f1": 0.33636151875145587
+        },
+        "copa": {
+            "acc": 0.55,
+            "acc_stderr": 0.04999999999999999
+        },
+        "hellaswag": {
+            "acc": 0.26379207329217286,
+            "acc_stderr": 0.00439787247185492,
+            "acc_norm": 0.2671778530173272,
+            "acc_norm_stderr": 0.004415816696303073
+        },
+        "rte": {
+            "acc": 0.49097472924187724,
+            "acc_stderr": 0.030091559826331334
+        },
+        "winogrande": {
+            "acc": 0.5146014206787688,
+            "acc_stderr": 0.014046492383275837
+        },
+        "storycloze_2016": {
+            "acc": 0.5189738107963656,
+            "acc_stderr": 0.011554104174019695
+        },
+        "boolq": {
+            "acc": 0.5850152905198777,
+            "acc_stderr": 0.008617716361921567
+        },
+        "arc_easy": {
+            "acc": 0.3303872053872054,
+            "acc_stderr": 0.009651430216428194,
+            "acc_norm": 0.3253367003367003,
+            "acc_norm_stderr": 0.00961342770899619
+        },
+        "arc_challenge": {
+            "acc": 0.1621160409556314,
+            "acc_stderr": 0.01077026738079835,
+            "acc_norm": 0.2090443686006826,
+            "acc_norm_stderr": 0.011882746987406446
+        },
+        "sciq": {
+            "acc": 0.547,
+            "acc_stderr": 0.015749255189977586,
+            "acc_norm": 0.534,
+            "acc_norm_stderr": 0.015782683329937614
+        },
+        "piqa": {
+            "acc": 0.573993471164309,
+            "acc_stderr": 0.011537375448519445,
+            "acc_norm": 0.5652883569096845,
+            "acc_norm_stderr": 0.011565943814308855
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..1c9d9b7e0562b346e5ab9a5ae2b1f069aca9a2af
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.324,0.014806864733738859,0
+anli_r2,acc,0.34,0.014987482264363937,0
+anli_r3,acc,0.335,0.013630871843821479,0
+arc_challenge,acc,0.1689419795221843,0.010949795652485024,0
+arc_challenge,acc_norm,0.2090443686006826,0.011882746987406448,0
+arc_easy,acc,0.32786195286195285,0.009632587076170018,0
+arc_easy,acc_norm,0.3085016835016835,0.009477472342978126,0
+boolq,acc,0.6146788990825688,0.008511930879680647,1
+cb,acc,0.48214285714285715,0.0673769750864465,1
+cb,f1,0.3356643356643356,,1
+copa,acc,0.57,0.049756985195624284,0
+hellaswag,acc,0.26538538139812784,0.004406358190678484,0
+hellaswag,acc_norm,0.2664807807209719,0.004412149415717922,0
+piqa,acc,0.5701849836779108,0.011550322268694083,0
+piqa,acc_norm,0.5522306855277476,0.011601999796866812,0
+rte,acc,0.5126353790613718,0.030086851767188564,0
+sciq,acc,0.561,0.015701131345400774,0
+sciq,acc_norm,0.554,0.015726771166750357,0
+storycloze_2016,acc,0.5227151256012827,0.01155049419200895,0
+winogrande,acc,0.5074980268350434,0.014050905521228573,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..d75598202abc14d33fe73d74b8fbe88f8131fce5
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-05_4shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.324,
+            "acc_stderr": 0.014806864733738859
+        },
+        "anli_r2": {
+            "acc": 0.34,
+            "acc_stderr": 0.014987482264363937
+        },
+        "anli_r3": {
+            "acc": 0.335,
+            "acc_stderr": 0.013630871843821479
+        },
+        "cb": {
+            "acc": 0.48214285714285715,
+            "acc_stderr": 0.0673769750864465,
+            "f1": 0.3356643356643356
+        },
+        "copa": {
+            "acc": 0.57,
+            "acc_stderr": 0.049756985195624284
+        },
+        "hellaswag": {
+            "acc": 0.26538538139812784,
+            "acc_stderr": 0.004406358190678484,
+            "acc_norm": 0.2664807807209719,
+            "acc_norm_stderr": 0.004412149415717922
+        },
+        "rte": {
+            "acc": 0.5126353790613718,
+            "acc_stderr": 0.030086851767188564
+        },
+        "winogrande": {
+            "acc": 0.5074980268350434,
+            "acc_stderr": 0.014050905521228573
+        },
+        "storycloze_2016": {
+            "acc": 0.5227151256012827,
+            "acc_stderr": 0.01155049419200895
+        },
+        "boolq": {
+            "acc": 0.6146788990825688,
+            "acc_stderr": 0.008511930879680647
+        },
+        "arc_easy": {
+            "acc": 0.32786195286195285,
+            "acc_stderr": 0.009632587076170018,
+            "acc_norm": 0.3085016835016835,
+            "acc_norm_stderr": 0.009477472342978126
+        },
+        "arc_challenge": {
+            "acc": 0.1689419795221843,
+            "acc_stderr": 0.010949795652485024,
+            "acc_norm": 0.2090443686006826,
+            "acc_norm_stderr": 0.011882746987406448
+        },
+        "sciq": {
+            "acc": 0.561,
+            "acc_stderr": 0.015701131345400774,
+            "acc_norm": 0.554,
+            "acc_norm_stderr": 0.015726771166750357
+        },
+        "piqa": {
+            "acc": 0.5701849836779108,
+            "acc_stderr": 0.011550322268694083,
+            "acc_norm": 0.5522306855277476,
+            "acc_norm_stderr": 0.011601999796866812
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4a8f00e9c7a4218d431a071bd583f38a11bd3d3f
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.314,0.014683991951087976,0
+anli_r2,acc,0.336,0.014944140233795023,0
+anli_r3,acc,0.32166666666666666,0.013490095282989521,0
+arc_challenge,acc,0.17747440273037543,0.01116513876964396,0
+arc_challenge,acc_norm,0.20819112627986347,0.011864866118448069,0
+arc_easy,acc,0.3265993265993266,0.009623047038267647,0
+arc_easy,acc_norm,0.31734006734006737,0.009550648343947771,0
+boolq,acc,0.6021406727828746,0.008560641169303364,1
+cb,acc,0.4107142857142857,0.0663363415035954,1
+cb,f1,0.286512928022362,,1
+copa,acc,0.6,0.049236596391733084,0
+hellaswag,acc,0.2657837084246166,0.004408468107262735,0
+hellaswag,acc_norm,0.2660824536944832,0.00441004753083503,0
+piqa,acc,0.5788900979325353,0.01151970105915149,0
+piqa,acc_norm,0.5680087051142546,0.01155740721010026,0
+rte,acc,0.4729241877256318,0.030052303463143706,0
+sciq,acc,0.562,0.01569721001969469,0
+sciq,acc_norm,0.552,0.015733516566347833,0
+storycloze_2016,acc,0.5173703901656868,0.011555452669106639,0
+winogrande,acc,0.5114443567482242,0.014048804199859332,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..3c6051e79fd8f1b461c8562564e9f2923b01ff55
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-13-57-07_3shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.314,
+            "acc_stderr": 0.014683991951087976
+        },
+        "anli_r2": {
+            "acc": 0.336,
+            "acc_stderr": 0.014944140233795023
+        },
+        "anli_r3": {
+            "acc": 0.32166666666666666,
+            "acc_stderr": 0.013490095282989521
+        },
+        "cb": {
+            "acc": 0.4107142857142857,
+            "acc_stderr": 0.0663363415035954,
+            "f1": 0.286512928022362
+        },
+        "copa": {
+            "acc": 0.6,
+            "acc_stderr": 0.049236596391733084
+        },
+        "hellaswag": {
+            "acc": 0.2657837084246166,
+            "acc_stderr": 0.004408468107262735,
+            "acc_norm": 0.2660824536944832,
+            "acc_norm_stderr": 0.00441004753083503
+        },
+        "rte": {
+            "acc": 0.4729241877256318,
+            "acc_stderr": 0.030052303463143706
+        },
+        "winogrande": {
+            "acc": 0.5114443567482242,
+            "acc_stderr": 0.014048804199859332
+        },
+        "storycloze_2016": {
+            "acc": 0.5173703901656868,
+            "acc_stderr": 0.011555452669106639
+        },
+        "boolq": {
+            "acc": 0.6021406727828746,
+            "acc_stderr": 0.008560641169303364
+        },
+        "arc_easy": {
+            "acc": 0.3265993265993266,
+            "acc_stderr": 0.009623047038267647,
+            "acc_norm": 0.31734006734006737,
+            "acc_norm_stderr": 0.009550648343947771
+        },
+        "arc_challenge": {
+            "acc": 0.17747440273037543,
+            "acc_stderr": 0.01116513876964396,
+            "acc_norm": 0.20819112627986347,
+            "acc_norm_stderr": 0.011864866118448069
+        },
+        "sciq": {
+            "acc": 0.562,
+            "acc_stderr": 0.01569721001969469,
+            "acc_norm": 0.552,
+            "acc_norm_stderr": 0.015733516566347833
+        },
+        "piqa": {
+            "acc": 0.5788900979325353,
+            "acc_stderr": 0.01151970105915149,
+            "acc_norm": 0.5680087051142546,
+            "acc_norm_stderr": 0.01155740721010026
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.csv b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.csv
new file mode 100644
index 0000000000000000000000000000000000000000..ae3c3a5b9db36118937f68430aa25d810401b801
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.csv
@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.344,0.015029633724408945,0
+anli_r2,acc,0.329,0.014865395385928355,0
+anli_r3,acc,0.35583333333333333,0.013826518748493322,0
+arc_challenge,acc,0.16467576791808874,0.010838369209479231,0
+arc_challenge,acc_norm,0.20733788395904437,0.011846905782971364,0
+arc_easy,acc,0.3421717171717172,0.009735236771958743,0
+arc_easy,acc_norm,0.32365319865319864,0.009600478182273768,0
+boolq,acc,0.6128440366972477,0.008519429207594412,1
+cb,acc,0.4642857142857143,0.06724777654937658,1
+cb,f1,0.3235431235431235,,1
+copa,acc,0.62,0.048783173121456316,0
+hellaswag,acc,0.2658832901812388,0.0044089948686501,0
+hellaswag,acc_norm,0.26558454491137223,0.004407413723383407,0
+piqa,acc,0.5669205658324266,0.01156086442315138,0
+piqa,acc_norm,0.5571273122959739,0.011589430503509102,0
+rte,acc,0.5523465703971119,0.02993107036293953,0
+sciq,acc,0.556,0.01571976816340209,0
+sciq,acc_norm,0.541,0.015766025737882165,0
+storycloze_2016,acc,0.5227151256012827,0.011550494192008948,0
+winogrande,acc,0.5011838989739542,0.014052446290529015,0
diff --git a/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.json b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ed0aeb956c130dacda2dad347e496e66a50f7f8
--- /dev/null
+++ b/1b11b51b5/evaluation/rankeval/lm1-1b1-1b5-results_lm-eval_global_step2891_2023-01-24-17-05-38_5shots.json
@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.344,
+            "acc_stderr": 0.015029633724408945
+        },
+        "anli_r2": {
+            "acc": 0.329,
+            "acc_stderr": 0.014865395385928355
+        },
+        "anli_r3": {
+            "acc": 0.35583333333333333,
+            "acc_stderr": 0.013826518748493322
+        },
+        "cb": {
+            "acc": 0.4642857142857143,
+            "acc_stderr": 0.06724777654937658,
+            "f1": 0.3235431235431235
+        },
+        "copa": {
+            "acc": 0.62,
+            "acc_stderr": 0.048783173121456316
+        },
+        "hellaswag": {
+            "acc": 0.2658832901812388,
+            "acc_stderr": 0.0044089948686501,
+            "acc_norm": 0.26558454491137223,
+            "acc_norm_stderr": 0.004407413723383407
+        },
+        "rte": {
+            "acc": 0.5523465703971119,
+            "acc_stderr": 0.02993107036293953
+        },
+        "winogrande": {
+            "acc": 0.5011838989739542,
+            "acc_stderr": 0.014052446290529015
+        },
+        "storycloze_2016": {
+            "acc": 0.5227151256012827,
+            "acc_stderr": 0.011550494192008948
+        },
+        "boolq": {
+            "acc": 0.6128440366972477,
+            "acc_stderr": 0.008519429207594412
+        },
+        "arc_easy": {
+            "acc": 0.3421717171717172,
+            "acc_stderr": 0.009735236771958743,
+            "acc_norm": 0.32365319865319864,
+            "acc_norm_stderr": 0.009600478182273768
+        },
+        "arc_challenge": {
+            "acc": 0.16467576791808874,
+            "acc_stderr": 0.010838369209479231,
+            "acc_norm": 0.20733788395904437,
+            "acc_norm_stderr": 0.011846905782971364
+        },
+        "sciq": {
+            "acc": 0.556,
+            "acc_stderr": 0.01571976816340209,
+            "acc_norm": 0.541,
+            "acc_norm_stderr": 0.015766025737882165
+        },
+        "piqa": {
+            "acc": 0.5669205658324266,
+            "acc_stderr": 0.01156086442315138,
+            "acc_norm": 0.5571273122959739,
+            "acc_norm_stderr": 0.011589430503509102
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}
\ No newline at end of file
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22aec3c7c06d07d906a0f3759cfd12291e79fbc7
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2b0a5e9bbd75335240e0c430c995de6ac3fca29421389c84857bdf4463ae8f9
+size 205568023
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05fe3a7aab1414c37c18b75f7c9d4b0ef54f5533
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d04f8ac5c56af391a9f159d5d569d59e898ef29ac3e7e792bcdefc2250399d7
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68b5a5d65b3152f66e7837f2c9086b2425f64736
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77d80d4285b5ff06ec97b5bb5de94f00379f9f596223fceda7470bc12be4a84c
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b9a540054b040afcfb6378ed049a7894387a0984
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfbcd133485529a92beae20ba11c6567662ef60a3f57b6a1a00947c4122a219c
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..26342616bfba731000568448e8b244f4e5d98c1c
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd6cfd1dc0f832ad65afbf6d7f99951139badabe8f69cc80210d422a8131ed0
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..94d242aed64a89f4cfae32ad2b37a7aee9ea7da9
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44200603a162d46736d985508c54790560d146f2d147d83e9ff81ba870f53219
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..10b732147f8c94c3d582902b38d7cf80cc4e622e
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5612151b82a425700ebe0a9b8280210e898dd33473d17ff55bbde198550de5c9
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..846a41efad03eb01bcbaf246b9c3fdb89db504ad
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32d52b724ba46e2971a63c4055a482bfcb3b4f5d2fe1488d5fd6a0f709f9d6c4
+size 205567970
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a2dfbdd60888ea1c13ca8130244a78c39c3a21b1
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48b1269d770d83435f042d3851dfcd4c63673b8588c9de52b9a23de6b69771a1
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb94a840b56fdce727783ce3b5490b3519659231
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a858112721b264028d00d60fa84214e06825d5c23c3be0d0d3fa6d42e5382fc
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b81efd537b24a34ebee2147004ea5824eda93e16
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edcebe23012aba667643b7f7a4e73618b059aa85aaaa1a3d5a9f228e9e82d575
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76a7dbd243009f1c23fe57dc39a9e4b0eb99f316
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03b9e8469c3755a926059e43ef772d639d2be9fa9e01cf45d380dfc852a1086d
+size 205568151
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9cc2f21ea7cd31a974eddb88e058a9f8799d1735
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e71a3ca96423104653b4d1426bed49e6461abcf7f538391abe84a0074048e98
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2a4769a0318a7e2fd882ebe6b07ccd2ecf6963c0
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6f8eb834ba8626603e5a3da5f6b442beaac6801f2c3812f5a5fa38cbd534cc1
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4eb402e8c88460d8a9a80ff4af0b6e08b5af5593
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e0381d9f10fd5fd7b9a6eed0f1e547fdbdf9511948f49e15846663e73740cbe
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1457592b2c5c623e71a48a18277a9db3601c612e
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f658750682ebf97feb3a2e649afe46dddf5c160f64e13d50ac479dbe458920e
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fa84d04c46db020c93f4b65a33aef6d4beb32ac7
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a32d8feef557393bd8484ce877601e9156a3d9fd6b8b29dcbf79118c5004cd6
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d84ab459b58585be2e831d8e9a93da14ff59ad45
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:311c335515ff9a76860cfe5d7cf23db2119f72f61e20591af489ccb399844472
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5d57463d9d69aadf410664db4363cb708009570b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e2905bd07d52733e776f27e51be835fd16d21e8449eb908e379b1f3e3871dde
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d98e1b90d18bfe2e698116383825d5e4271006bc
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21747d8cdc43db726b667cc5c0bbbc3ad3e12c29a89e5e7a93a828c3b5303958
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..788dd36204800912e57308cabefca0d062a39fc2
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fe1ce36517c7aaddef11d8271af7c03f54a3d4ec53b4298bde45e8972835a95
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d466b484bc19f9da3338cba5b5cf6a7dba6d6bc9
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d063c6b46ad206f2fa94bac533d215b5984f5b93e5ed7d297de665c3f26ec6a5
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4618feef8b4b3cafb3ece38c8a5552d018254e7b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68968347912f65d3a32ab158f500a9e2d5fcaaf3a3878985a389146894878699
+size 205568151
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3083e1f709de71478b467707eff34b9ccdf92f88
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bbb1c20b85b74a166af55a3529cfc08613ad7fbfde64830d936d521234b0e58
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..90eef1216a0d829a7e8c7d9e7f99e8d367fd244d
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d520a9846063d237e0e29c10813a8a6756fd170f9a59168016d268d3f1eed185
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e0a694f6b4899ed03d190750f4e22975cc248264
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72d2469bb8c348fbd073de8145ad9caa32ecdde727c7dafb12280b4f5754ecb5
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..05be7be3d0b5d3b656c1ace1147c63859992cb77
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a968aa6167c86d1d77a352816c4336529aef661c43c6e73476732566ef8e1455
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4fc646aaf06bc7c3b9a86fdfb9840e1a3760414d
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:249f043690bd3c9a26db77f9ad60604de493656f849de32918fdd63198150a44
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2fc65482c285769295d3b91791a0f2819d4cfde6
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502a228f4f4b793870ee3ec2f3781e640db218d5a922f02936b75d42abd7c9db
+size 205568290
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fc409508ad1751fd7114a31416db36ac7a304876
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06dc09ebc1480d55691b9c5b0c2ecae45c92b606c8d826ac3e55ad8dcb9e255b
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..51a8026758a68de27daa6c9dd53699312faea1de
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07993e0e382e601a0a38806b8d2ba6f5998a78c2d4b033f1003a7b0a2564aa3a
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..108257e45b324f2c758602a8f253bed9bbf0d783
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a64283bd4e5322e773db513a514ef89ef69c3efd1924ed71aab49a3724ca3507
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..af1c24e0d609bd5e4a8d4c9ccc0fa5b760bee940
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98ffa0ada8e5ecd1a687c54362c8ff31506b97edaad18ba2a8481c4363a151d
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..22aa08dfa481c8316d317e98831310db7196704c
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5593d66908fc0e476b9ea9ce3596054e6d5d6dd069d9f9534a82d61cc36f44ee
+size 205568215
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..35286a493b2920061f7d15e3e7b0026a1e15800b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f05709e0c17f47c692aa704416dc2ede8104b28034acde0a4e676b235fb53288
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac93f6b1223561ee3e9ea3929144f6c051447fd7
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0815ad4df7587a850029a83ed7ab04b62501b9a82f31ee352339999a5aa9de
+size 205568354
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c3d201b635af0fc10220dba6369b73fcb9c2152f
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0c134859c20d2c375bfe681cbdcce1044e15bc11493f58ecb499cb53593e67d
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd86496dea4b78ad5f39de0251666801281943c1
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ccc2b0b97b1f7e00daa8b6650a5a4c352ce093b51761f387fdec46f29d3ef2c
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c9b555b6348acb82d1038c8be96dbae843fbb6e0
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9d1cbe7c118ae2c185f36f32654c079f7732077ab55880f4af845f976a7b461
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fd6d330d0cf89e2563e4d76151baea1b03e01550
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbd1accbee3b393c74f171ca5d4667543dcb4938d87d6cb5e17255939eeb251f
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..16edc72d52e7eb95365336265f1616123be37bad
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ec86386f23e92b440a32bba6d544dca05044d204d66928e4286ecd3e1fcf52
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f203102e33458af4a5bd68a67b95a434636fdefc
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f59b17ce8f6dd156f10a706087f5cef2d5c82ffd87f42b7d08a2c4608100ecf
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..45f9ce803076f3695100f0c54c2428a2a8749d89
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e6275f6731942093b0d0f0a257b1d6a0e7a2f2704bbc68183754ae7786f6b42
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..667b0fd89d772988525334d40421779f9ed29d5b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f0665cc317c6addb326a41d2c902e67da8d6812b7bd5a48ede31cac708018f
+size 205568290
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0ad12f1ec96680c11925d4ff3b49f059f9a81568
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce1a1e4c70fad5fdde4faa4ebf4652f68201ad6ffc7e1e7e5cc403a27a8baff9
+size 205568023
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f4cbf209f2bae4427703747fe227e50f4f2b545b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:220600e1aa4bff89aa0ec3382ab59b32bf0c8623e98a81f4a1efc965188ad5a8
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dacd8127bb7ba15f99aa0b7622e8279d8d808ff3
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6899d3aefc28bc2518c933e51cf2ee2e1f5014b317d458e9d002f3c6c3d89cb
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..743cfaa0cffb8a3fed57ce32dd71b9d927a70fad
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c323ef3fb68d8ec3742dbf6ef43ea92c6a345a674b50349af1e4516c0a55c899
+size 205568290
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b04beb3f7974e8c61e16d59b3fb01925c514ef37
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c00c24197f4898ec6a202376f2b265147b607011588bd0ad9c91ac574ca7e86c
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..020873f62360b49df3d2c9812867047b60c96487
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3ef13252b0a7282cad095e1480565b54275a430e8c740ac2879839d9af7688f
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..609beeafa680f8eac650f74aa2a708a92a696e73
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66739e5490e90111de75dc3baf2afcec8973f55160a963e59fbd305e3c976476
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6b5d363f57730b0e9a029a0a954bf82f322b270d
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6baa4ccfe263c4481dda7b79057d63ad23501192a23ca2ba9a80ab42ca291300
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6cafefe3eee20a44efa74f6d37740b0df5188690
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c346e393064e5e47fabf2b843592f64237a19ddca77ac3540cba1e76647f7cd5
+size 205568290
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f7ef6b402d1c68c46f7348378d9a1f689bde6c0
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6643c9e38e325f9013f0f0137f1f63fc3c4ea0c37caf8d6c705d59da732d371
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b93545deb148ca3f434320ff18377ecaea685173
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2553e363deebad4db300c51f3d69ec7ac65483ca30ef492830a1b566966337c
+size 205568226
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..92556510adc821755fa5f44189818fb7b7753817
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef6f7ec43ffe1e3fdaf50ac1065e33d03d38744a3c91251c425ac4167904d42
+size 205568023
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..528865b60574e0c903915624dda6143b409c84d6
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f44935da61d1fdb468115538af6fd602a1b415f7a991472859a06b2be496e1
+size 205568162
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2d3bb317c0182d9d73869424ca4a14dd15801170
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5bcbbde7f9c9bb105d9c5de164c6009978754164da2135a6b2bf8786d6b1f9b
+size 205568098
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ee3f70269f203d180de75175a43731bbc99c38d2
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5edf35115c0772f7e7637c32a240177e10f983411829d6d48f7f78595b78444c
+size 205568290
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f6dc943e00ecc5ec9bb73631edad0e749ee75fe6
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd62cae361d9b4126322af372c720c01f4d7133097c4333407a97ea0753c0218
+size 205568034
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d368e1660ec04ec2486ae2fbd01bddda93f2c6e1
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079e117980e29ab110e52d30716e8f136deb375c0c37f3912f4fa456671624c1
+size 205568151
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c97057a51da642c111827c2bb678777d48843d7b
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fd6211c0a9913dc4a661309892048cce830cc42704cae3838694c6b0a3b031b
+size 205568151
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b552e8da549580018a3d39a482b5e3fa5e19c2f
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fe24c53862742554a65435c0481a89a82f112fd2fa933e9feffe6b6f17f957d
+size 205568151
diff --git a/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0f5e82a1384c3bd93a6b386411a63c69d596e9c2
--- /dev/null
+++ b/1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d7bc8a631eb8ad1b8ea4950b59c0c2af846e0c7f5ccf6fed7da67e8d46d08ff
+size 205568151
diff --git a/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f154d7bf4eaecd0d31af02246a5bc096dc285165
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_01-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:902fa5f73a997330a4147751f0361074324498294d578c59ca0dfd984ae8eadf
+size 187630851
diff --git a/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ed6f6b42f30bb498684b33fe9d5467261a9cb70f
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_03-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97f365eb5bda34333bb94476d8b1ba64dfc803f23c6fe4af9e6154bac741b8cc
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cea4e4daaea0bb85b67aa4bd8ff9cc136d48d03f
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_04-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8856df358d3da7afaf0f41f49e01841a99d0336f1f1f2f6974f05674f84efbeb
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..603ad5eb581b7819ce8a1aa64052b9a2604e4f7c
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_05-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97f47d04b1a2857e3813255d3c215ec2705bbaf5452d6f8570bf36f2098ec68b
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d92157a6734ad6d43ff544938a765a105b6ac214
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_06-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:840439c7f2a7dbbbca567de5d1bd8aca317e8f6606947220bd855df52ce04fe5
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0fb3cd97b1c1855b73361e782eae2de85e7ed689
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_07-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90d9d132a3a2c327e4e2e86326141cec7d207747b625afb934de24c4c3ebbc41
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7586779b2425396db5c5d32b5569aba8c94aca7c
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_08-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30fec2455be4062a9da5b562f28ff93ca7655643ac6725e855b37fb3e4c8205b
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..dba7fdae067d846b9e3feebf77f81a6066a4bd97
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_09-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c619f80935769d50b0a8a7e98ffdbfe700628cbb47c50b2ff01abe12dbff47d3
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fed9d2603358f43b60b84719191f75cb103a8b9
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_10-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e03f96d6fc11d6a39ad291af1dd88d0c10a1f7259621d28ed4816fee75cf7fd
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a730ebc814213ae9ad6184f6b909c2db9b56df6c
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_11-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13ce8cc21c965ad3dce3159f5d5d18cd776b14df194ff661027ce7998f87b77e
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f29b6e72fe11ac2a74184303ecaf29d4ed9a9ba4
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_12-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93abdf28ccb4a9df80b64ccd2a355e7d2feb399469a8a45592686ea2dc012109
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9309d6e98dd45b94bf6b3eb124fac1af7b5bff8d
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_13-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ab73ca8db00ef70616f5e3e54da8054ff8b19d9bdc718173e06f715b6ced1c4
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b8cbd56a86ab0433a890ed45e5f0da381e53f68
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_14-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3affbe928dc5663f3f156a7f3ff8ce528b57f37095c36cb3882c196316a1824c
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..995d4e3746b5142c0816bef39db1cfe20cbc11c6
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_15-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fff1bba512dfde4bc03ab3947d608d14ad582c08211e91d072dd1c0f123cb0a0
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d8332201d93852621699713587e7c802abbcdf6d
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_16-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b927c6060842cbdeeaeb5c7aecfacd00ef090e0677cf7418965202482186a773
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f0ca37883bff0af05b42632f2e41435c0bd95fea
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_17-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41e1b8ab15e5e1deac38efbcb95a329446bb28684684e6e3606a64ab24d90eb6
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..de01d57cb3029d8f20dea28930ed653b94915e96
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_18-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d011209a98ee0e880fc0984c5375eaf30099f566fcd9ce4e9ed594f7f42599b
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..649300d6dce6563f9bb20e41245f558d5e4256cd
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_19-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33000fae18a82bbd0e3fed63278c9018f4c7878039b1f3f6f9924d64b0e9b81f
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78840235613ccd532851f4c74e5366443ed4f14c
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_20-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efe49c9754cb5eb50d55010e9205daa7d24f60e9d1d99246dbb8da1d91343548
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..78df54a4773088e70d8a8c936c7c0b5c4f3ae94d
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_21-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc2e30ff07dc6c425395be085b154459199e86de8c071771af4f1b79242cd9a
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7dca01895fdf8143fa6b70a77c48b0eca7ebbcae
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_22-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c79e3eff0acf4a6fa4c8a4d589d1101f0b6750638336c737127ee9f8e9fbba7
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d2b4fa52b8d5cabbfcf3b41c8dad15d199961314
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_23-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58ca93efeaf51030429d7ec9ea9749c693d046ee9bd2fa249d57a5444551101b
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3fa98559c67ebfa25dc1c9a93bdd2a2989bd07a7
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_24-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e64dd7ed14690ce2af070ab074fc3a0994fe058267ed1ac1f2ccc4b165a0afba
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d42681bd3ca3a6a46cb7476c5737026381f0afa
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_25-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9f9ba9193e1f10ad641b362f7dea40e46975688c1dfe3ae93f5f474876fa357
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8197300cad2fdb2ca7692638eb0016d6159fd72e
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_26-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee697f287b710b350502a5ec014db5d05cc5b5a53d7ebc0a0d1558dfb54decb
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d18591eb037aa367e75306a99a93933ba8e6d205
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_27-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f7be82546bc75d04d99e9913435e0747fcc18515299aab68897df431e8a538d
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b924910aebfca4e2ce2547f45d60c8e1263514f4
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_28-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fff7c4b4eab4ec5ba2f2eef232947b1921f549881e0aef3255d0d3eadb1a9b53
+size 77121283
diff --git a/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt b/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d94ee4ab88956cfa6277029d846b5853e468f74e
--- /dev/null
+++ b/1b11b51b5/global_step2891/layer_30-model_00-model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cce4fac1665c424bd8d920b557272c5ce722d9b4aec7d69b83fa20977f2cb7ef
+size 8387
diff --git a/1b11b51b5/global_step2891/mp_rank_00_model_states.pt b/1b11b51b5/global_step2891/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d88b971836362755a14a2799bf0e3920689f3bea
--- /dev/null
+++ b/1b11b51b5/global_step2891/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5759ee7f0d1022f3b6d809d570f8733b50b1af59cd25cea6f78e852d945df62a
+size 43507
diff --git a/1b11b51b5/logs/2068467.err b/1b11b51b5/logs/2068467.err
new file mode 100644
index 0000000000000000000000000000000000000000..b06839107ffd894a8c4d30cf4734af65531b8289
--- /dev/null
+++ b/1b11b51b5/logs/2068467.err
@@ -0,0 +1,486 @@
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module scaled_upper_triang_masked_softmax_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module scaled_upper_triang_masked_softmax_cuda...
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module scaled_masked_softmax_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module scaled_masked_softmax_cuda...
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module fused_mix_prec_layer_norm_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module fused_mix_prec_layer_norm_cuda...
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+7: Successfully preprocessed all matching files.
+7: Successfully preprocessed all matching files.
+5: Successfully preprocessed all matching files.
+7: Successfully preprocessed all matching files.
+2: Successfully preprocessed all matching files.
+6: Successfully preprocessed all matching files.
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja...
+7: Building extension module utils...
+7: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+1: 
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+1: 
+1: 
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: 
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: 
+4: 
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: 
+4: 
+4: 
+4: 
+7: Loading extension module utils...
+6: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja...
+6: Building extension module utils...
+6: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+6: Loading extension module utils...
+3: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+3: Loading extension module utils...
+0: Loading extension module utils...
+3: Loading extension module utils...
+3: Loading extension module utils...
+3: Loading extension module utils...
+3: Loading extension module utils...
+3: Loading extension module utils...
+3: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+2: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+7: Loading extension module utils...
+1: Loading extension module utils...
+5: Loading extension module utils...
+6: Loading extension module utils...
+6: Loading extension module utils...
+5: Loading extension module utils...
+6: Loading extension module utils...
+5: Loading extension module utils...
+6: Loading extension module utils...
+1: Loading extension module utils...
+6: Loading extension module utils...
+6: Loading extension module utils...
+1: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+1: Loading extension module utils...
+5: Loading extension module utils...
+1: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+1: Loading extension module utils...
+1: Loading extension module utils...
+1: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...
+3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...
+3: 
+3: 
+3: 
+3: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils...Loading extension module utils...
+3: Loading extension module utils...
+3: 
+3: 
+3: 
+3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...
+3: 
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...
+1: 
+1: Loading extension module utils...Loading extension module utils...
+1: 
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: 
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...
+7: 
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...No modifications detected for re-loaded extension module utils, skipping build step...
+0: 
+0: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: 
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Loading extension module utils...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: 
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...
+4: 
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: 
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...
+5: 
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...
+5: 
+5: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...
+5: 
+5: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...
+6: 
+6: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings
+0:   warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings")
diff --git a/1b11b51b5/logs/2068467.out b/1b11b51b5/logs/2068467.out
new file mode 100644
index 0000000000000000000000000000000000000000..3291cd8054d933d8461146fc8782acf0234a01cf
--- /dev/null
+++ b/1b11b51b5/logs/2068467.out
@@ -0,0 +1,2150 @@
+Model parameters: d_model 1792 ffw_size 7168 kv_size 128 n_heads 14 n_layers 26
+Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 26 --hidden-size 1792 --num-attention-heads 14 --kv-channels 128 --ffn-hidden-size 7168 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 740_269 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --loss-scale 12 --clip-grad 1.0 --kill-switch-path kill-switch-1b1 --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 740_269 --lr-warmup-samples 7403 --clip-grad 1.0 --weight-decay 1e-1 --log-interval 10 --save-interval 1000 --eval-interval 1000 --eval-iters 1 --tensorboard-dir tensorboard_1b1 --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_1b1 --load checkpoints_1b1 --data-path /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document --data-impl mmap --split 949,50,1 --deepspeed --deepspeed_config ds_configs/2068467.json --zero-stage 0
+START 2068467: Thu Nov 24 20:22:00 EET 2022
+0: 
+0: 
+0: ======================= ROCm System Management Interface =======================
+0: ================================= Concise Info =================================
+0: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+0: 0    42.0c  91.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 1    47.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 2    46.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 3    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 4    44.0c  84.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 5    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 6    40.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 7    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: ================================================================================
+0: ============================= End of ROCm SMI Log ==============================
+6: 
+6: 
+6: ======================= ROCm System Management Interface =======================
+6: ================================= Concise Info =================================
+6: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+6: 0    47.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 1    50.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 2    47.0c  93.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 3    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 4    43.0c  99.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 5    51.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 6    36.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 7    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: ================================================================================
+6: ============================= End of ROCm SMI Log ==============================
+1: 
+1: 
+1: ======================= ROCm System Management Interface =======================
+1: ================================= Concise Info =================================
+1: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+1: 0    47.0c  84.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 2    38.0c  84.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 3    40.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 4    43.0c  85.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 5    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 6    41.0c  88.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 7    48.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: ================================================================================
+1: ============================= End of ROCm SMI Log ==============================
+3: 
+3: 
+3: ======================= ROCm System Management Interface =======================
+3: ================================= Concise Info =================================
+3: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+3: 0    40.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 2    39.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 3    48.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 4    43.0c  83.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 5    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 6    40.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 7    51.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: ================================================================================
+3: ============================= End of ROCm SMI Log ==============================
+7: 
+7: 
+7: ======================= ROCm System Management Interface =======================
+7: ================================= Concise Info =================================
+7: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+7: 0    40.0c  91.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 1    49.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 2    44.0c  95.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 3    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 4    45.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 5    47.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 6    40.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 7    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: ================================================================================
+7: ============================= End of ROCm SMI Log ==============================
+2: 
+2: 
+2: ======================= ROCm System Management Interface =======================
+2: ================================= Concise Info =================================
+2: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+2: 0    46.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 1    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 2    43.0c  84.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 3    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 4    49.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 5    49.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 6    40.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 7    51.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: ================================================================================
+2: ============================= End of ROCm SMI Log ==============================
+5: 
+5: 
+5: ======================= ROCm System Management Interface =======================
+5: ================================= Concise Info =================================
+5: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+5: 0    46.0c  92.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 1    48.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 2    38.0c  93.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 3    48.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 4    47.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 5    39.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 6    46.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 7    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: ================================================================================
+5: ============================= End of ROCm SMI Log ==============================
+4: 
+4: 
+4: ======================= ROCm System Management Interface =======================
+4: ================================= Concise Info =================================
+4: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+4: 0    41.0c  95.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 2    43.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 3    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 4    44.0c  85.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 5    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 6    41.0c  89.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 7    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: ================================================================================
+4: ============================= End of ROCm SMI Log ==============================
+5: Launching on nid005046 (5/8), master nid005000 port 9999, GPUs 8, CUDA: True
+3: Launching on nid005003 (3/8), master nid005000 port 9999, GPUs 8, CUDA: True
+2: Launching on nid005002 (2/8), master nid005000 port 9999, GPUs 8, CUDA: True
+7: Launching on nid005048 (7/8), master nid005000 port 9999, GPUs 8, CUDA: True
+6: Launching on nid005047 (6/8), master nid005000 port 9999, GPUs 8, CUDA: True
+4: Launching on nid005004 (4/8), master nid005000 port 9999, GPUs 8, CUDA: True
+0: Launching on nid005000 (0/8), master nid005000 port 9999, GPUs 8, CUDA: True
+1: Launching on nid005001 (1/8), master nid005000 port 9999, GPUs 8, CUDA: True
+0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 
+0: accumulate and all-reduce gradients in fp32 for bfloat16 data type.
+0: using torch.bfloat16 for parameters ...
+0: ------------------------ arguments ------------------------
+0:   abort_on_unmet_fused_kernel_constraints ......... False
+0:   accumulate_allreduce_grads_in_fp32 .............. True
+0:   adam_beta1 ...................................... 0.9
+0:   adam_beta2 ...................................... 0.999
+0:   adam_eps ........................................ 1e-08
+0:   adlr_autoresume ................................. False
+0:   adlr_autoresume_interval ........................ 1000
+0:   apply_query_key_layer_scaling ................... True
+0:   apply_residual_connection_post_layernorm ........ False
+0:   attention_dropout ............................... 0.1
+0:   attention_softmax_in_fp32 ....................... False
+0:   bert_binary_head ................................ True
+0:   bert_load ....................................... None
+0:   bf16 ............................................ True
+0:   bias_dropout_fusion ............................. True
+0:   bias_gelu_fusion ................................ True
+0:   biencoder_projection_dim ........................ 0
+0:   biencoder_shared_query_context_model ............ False
+0:   block_data_path ................................. None
+0:   checkpoint_activations .......................... False
+0:   checkpoint_in_cpu ............................... False
+0:   checkpoint_num_layers ........................... 1
+0:   clip_grad ....................................... 1.0
+0:   codecarbon_dir .................................. None
+0:   consumed_train_samples .......................... 0
+0:   consumed_train_tokens ........................... 0
+0:   consumed_valid_samples .......................... 0
+0:   contigious_checkpointing ........................ False
+0:   cpu_optimizer ................................... False
+0:   cpu_torch_adam .................................. False
+0:   curriculum_learning ............................. False
+0:   data_impl ....................................... mmap
+0:   data_parallel_size .............................. 64
+0:   data_path ....................................... ['/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document']
+0:   dataloader_type ................................. single
+0:   DDP_impl ........................................ local
+0:   decoder_seq_length .............................. None
+0:   deepscale ....................................... False
+0:   deepscale_config ................................ None
+0:   deepspeed ....................................... True
+0:   deepspeed_activation_checkpointing .............. False
+0:   deepspeed_config ................................ ds_configs/2068467.json
+0:   deepspeed_mpi ................................... False
+0:   distribute_checkpointed_activations ............. False
+0:   distributed_backend ............................. nccl
+0:   embed_layernorm ................................. False
+0:   embedding_path .................................. None
+0:   encoder_seq_length .............................. 2048
+0:   eod_mask_loss ................................... False
+0:   eval_interval ................................... 1000
+0:   eval_iters ...................................... 1
+0:   eval_only ....................................... None
+0:   evidence_data_path .............................. None
+0:   exit_duration_in_mins ........................... None
+0:   exit_interval ................................... None
+0:   ffn_hidden_size ................................. 7168
+0:   finetune ........................................ False
+0:   fp16 ............................................ False
+0:   fp16_lm_cross_entropy ........................... False
+0:   fp32_residual_connection ........................ False
+0:   gigaflos_no_embeds .............................. 0
+0:   global_batch_size ............................... 256
+0:   glu_activation .................................. None
+0:   hidden_dropout .................................. 0.1
+0:   hidden_size ..................................... 1792
+0:   hysteresis ...................................... 2
+0:   ict_head_size ................................... None
+0:   ict_load ........................................ None
+0:   img_dim ......................................... 224
+0:   indexer_batch_size .............................. 128
+0:   indexer_log_interval ............................ 1000
+0:   inference ....................................... False
+0:   init_method_std ................................. 0.02
+0:   init_method_xavier_uniform ...................... False
+0:   initial_loss_scale .............................. 4294967296
+0:   kill_switch_path ................................ kill-switch-1b1
+0:   kv_channels ..................................... 128
+0:   layer_norm_fusion ............................... True
+0:   layernorm_epsilon ............................... 1e-05
+0:   lazy_mpu_init ................................... None
+0:   load ............................................ checkpoints_1b1
+0:   local_rank ...................................... None
+0:   log_batch_size_to_tensorboard ................... True
+0:   log_interval .................................... 10
+0:   log_learning_rate_to_tensorboard ................ True
+0:   log_level ....................................... None
+0:   log_level_replica ............................... None
+0:   log_loss_scale_to_tensorboard ................... True
+0:   log_num_zeros_in_grad ........................... False
+0:   log_params_norm ................................. False
+0:   log_path ........................................ None
+0:   log_timers_to_tensorboard ....................... True
+0:   log_validation_ppl_to_tensorboard ............... True
+0:   loss_on_targets_only ............................ False
+0:   loss_scale ...................................... 12.0
+0:   loss_scale_window ............................... 1000
+0:   lr .............................................. 0.0002
+0:   lr_decay_iters .................................. None
+0:   lr_decay_samples ................................ 740269
+0:   lr_decay_style .................................. cosine
+0:   lr_decay_tokens ................................. None
+0:   lr_warmup_fraction .............................. None
+0:   lr_warmup_iters ................................. 0
+0:   lr_warmup_samples ............................... 7403
+0:   make_vocab_size_divisible_by .................... 128
+0:   mask_prob ....................................... 0.15
+0:   masked_softmax_fusion ........................... True
+0:   max_position_embeddings ......................... 2048
+0:   mean_noise_span_length .......................... None
+0:   memory_centric_tiled_linear ..................... False
+0:   merge_file ...................................... gpt2/merges.txt
+0:   micro_batch_size ................................ 4
+0:   min_loss_scale .................................. 1.0
+0:   min_lr .......................................... 2e-05
+0:   mmap_warmup ..................................... False
+0:   no_load_optim ................................... None
+0:   no_load_rng ..................................... None
+0:   no_save_optim ................................... None
+0:   no_save_rng ..................................... None
+0:   noise_density ................................... None
+0:   num_attention_heads ............................. 14
+0:   num_channels .................................... 3
+0:   num_classes ..................................... 1000
+0:   num_layers ...................................... 26
+0:   num_layers_per_virtual_pipeline_stage ........... None
+0:   num_workers ..................................... 2
+0:   onnx_safe ....................................... None
+0:   openai_gelu ..................................... False
+0:   optimizer ....................................... adam
+0:   optimizer_fusion ................................ True
+0:   override_lr_scheduler ........................... False
+0:   pad_vocab_size_to ............................... None
+0:   params_dtype .................................... torch.bfloat16
+0:   partition_activations ........................... False
+0:   patch_dim ....................................... 16
+0:   pipeline_model_parallel_size .................... 1
+0:   position_embedding_type ......................... PositionEmbeddingType.absolute
+0:   pp_partition_method ............................. None
+0:   profile_backward ................................ False
+0:   query_in_block_prob ............................. 0.1
+0:   rampup_batch_size ............................... None
+0:   rank ............................................ 0
+0:   remote_device ................................... none
+0:   reset_attention_mask ............................ False
+0:   reset_position_ids .............................. False
+0:   retriever_report_topk_accuracies ................ []
+0:   retriever_score_scaling ......................... False
+0:   retriever_seq_length ............................ 256
+0:   reweight_loss_based_on_position_frequency ....... False
+0:   sample_rate ..................................... 1.0
+0:   save ............................................ checkpoints_1b1
+0:   save_interval ................................... 1000
+0:   scatter_gather_tensors_in_pipeline .............. True
+0:   scattered_embeddings ............................ False
+0:   seed ............................................ 1234
+0:   seq_length ...................................... 2048
+0:   sgd_momentum .................................... 0.9
+0:   short_seq_prob .................................. 0.1
+0:   skip_train_iteration_range ...................... None
+0:   split ........................................... 949,50,1
+0:   split_transformers .............................. False
+0:   sync_tp_duplicated_parameters ................... False
+0:   synchronize_each_layer .......................... False
+0:   tensor_model_parallel_size ...................... 1
+0:   tensorboard_dir ................................. tensorboard_1b1
+0:   tensorboard_log_interval ........................ 1
+0:   tensorboard_queue_size .......................... 5
+0:   test_weighted_split_names ....................... None
+0:   test_weighted_split_paths ....................... None
+0:   test_weighted_split_paths_path .................. None
+0:   test_weighted_split_splits ...................... None
+0:   test_weighted_split_weights ..................... None
+0:   tile_factor ..................................... 1
+0:   titles_data_path ................................ None
+0:   tokenizer_name_or_path .......................... None
+0:   tokenizer_type .................................. GPT2BPETokenizer
+0:   train_iters ..................................... None
+0:   train_samples ................................... 740269
+0:   train_tokens .................................... None
+0:   train_weighted_split_paths ...................... None
+0:   train_weighted_split_paths_path ................. None
+0:   universal_checkpoint ............................ False
+0:   use_bnb_optimizer ............................... False
+0:   use_checkpoint_lr_scheduler ..................... False
+0:   use_contiguous_buffers_in_ddp ................... True
+0:   use_cpu_initialization .......................... None
+0:   use_one_sent_docs ............................... False
+0:   use_pin_memory .................................. False
+0:   valid_num_workers ............................... 2
+0:   valid_weighted_split_names ...................... None
+0:   valid_weighted_split_paths ...................... None
+0:   valid_weighted_split_paths_path ................. None
+0:   valid_weighted_split_splits ..................... None
+0:   valid_weighted_split_weights .................... None
+0:   virtual_pipeline_model_parallel_size ............ None
+0:   vocab_extra_ids ................................. 0
+0:   vocab_file ...................................... gpt2/vocab.json
+0:   weight_decay .................................... 0.1
+0:   world_size ...................................... 64
+0:   zero_allgather_bucket_size ...................... 0.0
+0:   zero_contigious_gradients ....................... False
+0:   zero_reduce_bucket_size ......................... 0.0
+0:   zero_reduce_scatter ............................. False
+0:   zero_stage ...................................... 0
+0: -------------------- end of arguments ---------------------
+0: setting number of micro-batches to constant 1
+0: > building GPT2BPETokenizer tokenizer ...
+0:  > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
+0: DeepSpeed general environment info:
+0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch']
+0: torch version .................... 1.13.0+rocm5.2
+0: torch cuda version ............... None
+0: torch hip version ................ 5.2.21151-afdc89f8
+0: nvcc version ..................... None
+0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed']
+0: deepspeed info ................... 0.7.5, unknown, unknown
+0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1
+0: **** Git info for Megatron: git_hash=unknown git_branch=unknown ****
+0: > initializing torch distributed ...
+0: [2022-11-24 20:23:07,768] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+7: > setting tensorboard ...
+0: > initializing tensor model parallel with size 1
+0: > initializing pipeline model parallel with size 1
+0: > setting random seeds to 1234 ...
+0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234
+0: > compiling dataset index builder ...
+0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data'
+0: make: Nothing to be done for 'default'.
+0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data'
+0: >>> done with dataset index builder. Compilation time: 0.100 seconds
+0: > compiling and loading fused kernels ...
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 87
+0: ninja: no work to do.
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 63
+0: [1/1] c++ scaled_masked_softmax_hip.cuda.o scaled_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o scaled_masked_softmax_cuda.so
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 67
+0: ninja: no work to do.
+0: >>> done with compiling and loading fused kernels. Compilation time: 19.557 seconds
+0: time to initialize megatron (seconds): 36.732
+0: [after megatron is initialized] datetime: 2022-11-24 20:23:32 
+0: building GPT model ...
+0: [2022-11-24 20:23:32,784] [INFO] [utils.py:827:see_memory_usage] Before Building Model
+0: [2022-11-24 20:23:32,785] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB         Max_MA 0.0 GB         CA 0.0 GB         Max_CA 0 GB 
+0: [2022-11-24 20:23:32,785] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.17 GB, percent = 6.0%
+0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None
+0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi
+0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4
+0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63}
+0: [2022-11-24 20:23:34,820] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer
+0: stage=0 layers=33
+0:      0: _to_float16
+0:      1: EmbeddingPipe
+0:      2: <lambda>
+0:      3: ParallelTransformerLayerPipe
+0:      4: ParallelTransformerLayerPipe
+0:      5: ParallelTransformerLayerPipe
+0:      6: ParallelTransformerLayerPipe
+0:      7: ParallelTransformerLayerPipe
+0:      8: ParallelTransformerLayerPipe
+0:      9: ParallelTransformerLayerPipe
+0:     10: ParallelTransformerLayerPipe
+0:     11: ParallelTransformerLayerPipe
+0:     12: ParallelTransformerLayerPipe
+0:     13: ParallelTransformerLayerPipe
+0:     14: ParallelTransformerLayerPipe
+0:     15: ParallelTransformerLayerPipe
+0:     16: ParallelTransformerLayerPipe
+0:     17: ParallelTransformerLayerPipe
+0:     18: ParallelTransformerLayerPipe
+0:     19: ParallelTransformerLayerPipe
+0:     20: ParallelTransformerLayerPipe
+0:     21: ParallelTransformerLayerPipe
+0:     22: ParallelTransformerLayerPipe
+0:     23: ParallelTransformerLayerPipe
+0:     24: ParallelTransformerLayerPipe
+0:     25: ParallelTransformerLayerPipe
+0:     26: ParallelTransformerLayerPipe
+0:     27: ParallelTransformerLayerPipe
+0:     28: ParallelTransformerLayerPipe
+0:     29: undo
+0:     30: MixedFusedLayerNorm
+0:     31: EmbeddingPipe
+0:     32: float16_to_fp32
+0:   loss: CrossEntropy
+0: [2022-11-24 20:23:35,450] [INFO] [utils.py:827:see_memory_usage] After Building Model
+0: [2022-11-24 20:23:35,450] [INFO] [utils.py:828:see_memory_usage] MA 2.05 GB         Max_MA 2.05 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2022-11-24 20:23:35,450] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.19 GB, percent = 6.0%
+0: setting training iterations to 2891
+0: > learning rate decay style: cosine
+0: DeepSpeed is enabled.
+0: [2022-11-24 20:23:35,453] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown
+0: [2022-11-24 20:23:48,855] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+0: [2022-11-24 20:23:48,855] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer
+0: [2022-11-24 20:23:48,855] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer
+0: [2022-11-24 20:23:48,873] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam
+0: [2022-11-24 20:23:48,873] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer
+0: [2022-11-24 20:23:48,914] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer
+0: [2022-11-24 20:23:48,915] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB         Max_MA 2.06 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2022-11-24 20:23:48,915] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.88 GB, percent = 6.1%
+7: ninja: no work to do.
+7: Time to load utils op: 0.21609926223754883 seconds
+6: ninja: no work to do.
+6: Time to load utils op: 0.1269679069519043 seconds
+3: Time to load utils op: 0.311692476272583 seconds
+3: Time to load utils op: 0.20229244232177734 seconds
+3: Time to load utils op: 0.202559232711792 seconds
+3: Time to load utils op: 0.20256304740905762 seconds
+3: Time to load utils op: 0.20277762413024902 seconds
+3: Time to load utils op: 0.20265674591064453 seconds
+3: Time to load utils op: 0.20296168327331543 seconds
+3: Time to load utils op: 0.20312070846557617 seconds
+0: Time to load utils op: 0.20792245864868164 seconds
+0: Time to load utils op: 0.20650672912597656 seconds
+0: Time to load utils op: 0.20940756797790527 seconds
+0: Time to load utils op: 0.20720505714416504 seconds
+0: Time to load utils op: 0.2080376148223877 seconds
+0: Time to load utils op: 0.20946335792541504 seconds
+0: Time to load utils op: 0.3074929714202881 seconds
+0: Time to load utils op: 0.20802569389343262 seconds
+7: Time to load utils op: 0.20307350158691406 seconds
+7: Time to load utils op: 0.2023029327392578 seconds
+7: Time to load utils op: 0.20310330390930176 seconds
+7: Time to load utils op: 0.20270156860351562 seconds
+7: Time to load utils op: 0.20250558853149414 seconds
+7: Time to load utils op: 0.2032179832458496 seconds
+6: Time to load utils op: 0.20524263381958008 seconds
+7: Time to load utils op: 0.20299482345581055 seconds
+6: Time to load utils op: 0.20513343811035156 seconds
+6: Time to load utils op: 0.20574712753295898 seconds
+6: Time to load utils op: 0.20684528350830078 seconds
+6: Time to load utils op: 0.20530319213867188 seconds
+6: Time to load utils op: 0.20628738403320312 seconds
+6: Time to load utils op: 0.20551657676696777 seconds
+2: Time to load utils op: 0.21174883842468262 secondsTime to load utils op: 0.2110891342163086 seconds
+2: 
+2: Time to load utils op: 0.21237683296203613 seconds
+2: Time to load utils op: 0.2117292881011963 seconds
+2: Time to load utils op: 0.212723970413208 secondsTime to load utils op: 0.21134257316589355 secondsTime to load utils op: 0.21212434768676758 seconds
+2: 
+2: 
+2: Time to load utils op: 0.21165919303894043 seconds
+5: Time to load utils op: 0.21256446838378906 seconds
+5: Time to load utils op: 0.21172618865966797 seconds
+5: Time to load utils op: 0.21220660209655762 seconds
+5: Time to load utils op: 0.2125389575958252 seconds
+5: Time to load utils op: 0.2125682830810547 secondsTime to load utils op: 0.21261978149414062 seconds
+5: 
+5: Time to load utils op: 0.21309399604797363 seconds
+5: Time to load utils op: 0.21202683448791504 seconds
+1: Time to load utils op: 0.21275568008422852 seconds
+1: Time to load utils op: 0.21276402473449707 seconds
+1: Time to load utils op: 0.21280741691589355 seconds
+1: Time to load utils op: 0.21281003952026367 seconds
+1: Time to load utils op: 0.21282672882080078 secondsTime to load utils op: 0.2128286361694336 seconds
+1: 
+1: Time to load utils op: 0.2128441333770752 seconds
+1: Time to load utils op: 0.21284055709838867 seconds
+4: Time to load utils op: 0.21168279647827148 seconds
+4: Time to load utils op: 0.21170520782470703 secondsTime to load utils op: 0.2117021083831787 seconds
+4: 
+4: Time to load utils op: 0.21172142028808594 seconds
+4: Time to load utils op: 0.21173739433288574 seconds
+4: Time to load utils op: 0.21173787117004395 seconds
+4: Time to load utils op: 0.21174240112304688 secondsTime to load utils op: 0.2117459774017334 seconds
+4: 
+0: [2022-11-24 20:23:49,256] [INFO] [utils.py:827:see_memory_usage] before initializing group 0
+0: [2022-11-24 20:23:49,256] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB         Max_MA 2.04 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2022-11-24 20:23:49,256] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.88 GB, percent = 6.1%
+3: Time to load utils op: 0.0005133152008056641 seconds
+3: Time to load utils op: 0.0005700588226318359 seconds
+3: Time to load utils op: 0.0004372596740722656 seconds
+3: Time to load utils op: 0.0004811286926269531 seconds
+3: Time to load utils op: 0.0004782676696777344 seconds
+3: Time to load utils op: 0.0004642009735107422 seconds
+3: Time to load utils op: 0.0006365776062011719 seconds
+3: Time to load utils op: 0.0006463527679443359 seconds
+1: Time to load utils op: 0.0009274482727050781 seconds
+1: Time to load utils op: 0.0008521080017089844 seconds
+0: [2022-11-24 20:23:49,507] [INFO] [utils.py:827:see_memory_usage] after initializing group 0
+1: Time to load utils op: 0.0009047985076904297 seconds
+1: Time to load utils op: 0.0006520748138427734 seconds
+1: Time to load utils op: 0.0009334087371826172 seconds
+1: Time to load utils op: 0.0010607242584228516 seconds
+1: Time to load utils op: 0.0009770393371582031 seconds
+1: Time to load utils op: 0.0006129741668701172 seconds
+0: [2022-11-24 20:23:49,508] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB         Max_MA 4.24 GB         CA 5.44 GB         Max_CA 5 GB 
+0: [2022-11-24 20:23:49,508] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.89 GB, percent = 6.1%
+7: Time to load utils op: 0.0005197525024414062 seconds
+7: Time to load utils op: 0.0005347728729248047 seconds
+7: Time to load utils op: 0.0005235671997070312 seconds
+7: Time to load utils op: 0.0005702972412109375 seconds
+7: Time to load utils op: 0.000568389892578125 seconds
+7: Time to load utils op: 0.0006320476531982422 seconds
+7: Time to load utils op: 0.0005376338958740234 seconds
+0: Time to load utils op: 0.0005240440368652344 seconds
+0: Time to load utils op: 0.00043487548828125 seconds
+0: Time to load utils op: 0.0005767345428466797 seconds
+0: Time to load utils op: 0.0005452632904052734 seconds
+7: Time to load utils op: 0.00046181678771972656 seconds
+0: Time to load utils op: 0.00041222572326660156 seconds
+0: Time to load utils op: 0.00042128562927246094 seconds
+0: Time to load utils op: 0.0004417896270751953 seconds
+2: Time to load utils op: 0.0007309913635253906 seconds
+2: Time to load utils op: 0.0009577274322509766 seconds
+2: Time to load utils op: 0.0011696815490722656 seconds
+2: Time to load utils op: 0.0011599063873291016 seconds
+2: Time to load utils op: 0.0011050701141357422 seconds
+2: Time to load utils op: 0.0009145736694335938 seconds
+2: Time to load utils op: 0.0010519027709960938 seconds
+2: Time to load utils op: 0.0010685920715332031 seconds
+4: Time to load utils op: 0.0010120868682861328 seconds
+4: Time to load utils op: 0.001016855239868164 seconds
+4: Time to load utils op: 0.0012238025665283203 seconds
+4: Time to load utils op: 0.0012431144714355469 seconds
+4: Time to load utils op: 0.0013909339904785156 seconds
+4: Time to load utils op: 0.0013432502746582031 seconds
+4: Time to load utils op: 0.0013623237609863281 seconds
+4: Time to load utils op: 0.0013179779052734375 seconds
+5: Time to load utils op: 0.0011153221130371094 seconds
+5: Time to load utils op: 0.0013453960418701172 seconds
+6: Time to load utils op: 0.0004634857177734375 seconds
+5: Time to load utils op: 0.0017385482788085938 seconds
+6: Time to load utils op: 0.0004413127899169922 seconds
+5: Time to load utils op: 0.0017087459564208984 secondsTime to load utils op: 0.0018095970153808594 seconds
+6: Time to load utils op: 0.0004107952117919922 seconds
+5: 
+5: Time to load utils op: 0.0016875267028808594 secondsTime to load utils op: 0.00173187255859375 seconds
+5: 
+5: Time to load utils op: 0.0018622875213623047 seconds
+6: Time to load utils op: 0.0005204677581787109 seconds
+6: Time to load utils op: 0.00041556358337402344 seconds
+6: Time to load utils op: 0.0004119873046875 seconds
+6: Time to load utils op: 0.00040650367736816406 seconds
+6: Time to load utils op: 0.0005161762237548828 seconds
+0: [2022-11-24 20:23:49,541] [INFO] [utils.py:827:see_memory_usage] before initializing group 1
+0: [2022-11-24 20:23:49,542] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB         Max_MA 4.24 GB         CA 5.44 GB         Max_CA 5 GB 
+0: [2022-11-24 20:23:49,542] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.02 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,574] [INFO] [utils.py:827:see_memory_usage] after initializing group 1
+0: [2022-11-24 20:23:49,575] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,575] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.02 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,606] [INFO] [utils.py:827:see_memory_usage] before initializing group 2
+0: [2022-11-24 20:23:49,606] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,606] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.03 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,641] [INFO] [utils.py:827:see_memory_usage] after initializing group 2
+0: [2022-11-24 20:23:49,641] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,641] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.03 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,672] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer
+0: [2022-11-24 20:23:49,673] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,673] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.03 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,709] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer
+0: [2022-11-24 20:23:49,709] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB         Max_MA 6.32 GB         CA 8.34 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,710] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.04 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,741] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer
+0: [2022-11-24 20:23:49,741] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB         Max_MA 6.32 GB         CA 8.34 GB         Max_CA 8 GB 
+0: [2022-11-24 20:23:49,741] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.04 GB, percent = 6.2%
+0: [2022-11-24 20:23:49,741] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam
+0: [2022-11-24 20:23:49,742] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler
+0: [2022-11-24 20:23:49,742] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <megatron.learning_rates.AnnealingLR object at 0x146fb5b82fd0>
+0: [2022-11-24 20:23:49,742] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0, 0.0, 0.0], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)]
+0: [2022-11-24 20:23:49,742] [INFO] [config.py:1007:print] DeepSpeedEngine configuration:
+0: [2022-11-24 20:23:49,742] [INFO] [config.py:1011:print]   activation_checkpointing_config  {
+0:     "partition_activations": false, 
+0:     "contiguous_memory_optimization": false, 
+0:     "cpu_checkpointing": false, 
+0:     "number_checkpoints": null, 
+0:     "synchronize_checkpoint_boundary": false, 
+0:     "profile": false
+0: }
+0: [2022-11-24 20:23:49,742] [INFO] [config.py:1011:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}
+0: [2022-11-24 20:23:49,742] [INFO] [config.py:1011:print]   amp_enabled .................. False
+0: [2022-11-24 20:23:49,742] [INFO] [config.py:1011:print]   amp_params ................... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   autotuning_config ............ {
+0:     "enabled": false, 
+0:     "start_step": null, 
+0:     "end_step": null, 
+0:     "metric_path": null, 
+0:     "arg_mappings": null, 
+0:     "metric": "throughput", 
+0:     "model_info": null, 
+0:     "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", 
+0:     "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", 
+0:     "overwrite": true, 
+0:     "fast": true, 
+0:     "start_profile_step": 3, 
+0:     "end_profile_step": 5, 
+0:     "tuner_type": "gridsearch", 
+0:     "tuner_early_stopping": 5, 
+0:     "tuner_num_trials": 50, 
+0:     "model_info_path": null, 
+0:     "mp_size": 1, 
+0:     "max_train_batch_size": null, 
+0:     "min_train_batch_size": 1, 
+0:     "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+0:     "min_train_micro_batch_size_per_gpu": 1, 
+0:     "num_tuning_micro_batch_sizes": 3
+0: }
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   bfloat16_enabled ............. True
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   checkpoint_parallel_write_pipeline  False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   checkpoint_tag_validation_enabled  True
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   checkpoint_tag_validation_fail  False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x146fb5b82d30>
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   communication_data_type ...... None
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa
+0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   curriculum_enabled ........... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   curriculum_params ............ False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   dataloader_drop_last ......... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   disable_allgather ............ False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   dump_state ................... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   dynamic_loss_scale_args ...... None
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_enabled ........... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_gas_boundary_resolution  1
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_layer_name ........ bert.encoder.layer
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_layer_num ......... 0
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_max_iter .......... 100
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_stability ......... 1e-06
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_tol ............... 0.01
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   eigenvalue_verbose ........... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   elasticity_enabled ........... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   flops_profiler_config ........ {
+0:     "enabled": false, 
+0:     "profile_step": 1, 
+0:     "module_depth": -1, 
+0:     "top_modules": 1, 
+0:     "detailed": true, 
+0:     "output_file": null
+0: }
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   fp16_auto_cast ............... None
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   fp16_enabled ................. False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   fp16_master_weights_and_gradients  False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   global_rank .................. 0
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   gradient_accumulation_steps .. 1
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   gradient_clipping ............ 1.0
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   gradient_predivide_factor .... 1.0
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   initial_dynamic_scale ........ 1
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   load_universal_checkpoint .... False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   loss_scale ................... 1.0
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   memory_breakdown ............. False
+0: [2022-11-24 20:23:49,743] [INFO] [config.py:1011:print]   monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x146fb5b82ca0>
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   nebula_config ................ {
+0:     "enabled": false, 
+0:     "persistent_storage_path": null, 
+0:     "persistent_time_interval": 100, 
+0:     "num_of_version_in_retention": 2, 
+0:     "enable_nebula_load": true, 
+0:     "load_path": null
+0: }
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   optimizer_legacy_fusion ...... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   optimizer_name ............... None
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   optimizer_params ............. None
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   pld_enabled .................. False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   pld_params ................... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   prescale_gradients ........... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   scheduler_name ............... None
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   scheduler_params ............. None
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   sparse_attention ............. None
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   sparse_gradients_enabled ..... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   steps_per_print .............. 2000
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   train_batch_size ............. 256
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   train_micro_batch_size_per_gpu  4
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   use_node_local_storage ....... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   wall_clock_breakdown ......... False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   world_size ................... 64
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   zero_allow_untested_optimizer  False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   zero_enabled ................. False
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:1011:print]   zero_optimization_stage ...... 0
+0: [2022-11-24 20:23:49,744] [INFO] [config.py:996:print_user_config]   json = {
+0:     "train_micro_batch_size_per_gpu": 4, 
+0:     "train_batch_size": 256, 
+0:     "gradient_clipping": 1.0, 
+0:     "zero_optimization": {
+0:         "stage": 0
+0:     }, 
+0:     "bf16": {
+0:         "enabled": true
+0:     }, 
+0:     "steps_per_print": 2.000000e+03, 
+0:     "wall_clock_breakdown": false
+0: }
+0: Time to load utils op: 0.0004105567932128906 seconds
+0: [2022-11-24 20:23:49,745] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4
+0: [2022-11-24 20:23:49,766] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=33 [0, 33) STAGE_PARAMS=1096338432 (1096.338M) TOTAL_PARAMS=1096338432 (1096.338M) UNIQUE_PARAMS=1096338432 (1096.338M)
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: WARNING: could not find the metadata file checkpoints_1b1 
+0: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0:     will not load any checkpoints and will start from random
+7: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,770] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+4: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+0: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+6: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+3: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+5: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+2: [2022-11-24 20:23:49,771] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,772] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+1: [2022-11-24 20:23:49,772] [WARNING] [engine.py:2581:load_checkpoint] Unable to find latest file at checkpoints_1b1/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint.
+7: time (ms) | load-checkpoint: 7.68
+0: estimated model parameters: 1.096338432
+0: estimated model parameters without embeddings: 1.002523648
+0: [after model, optimizer, and learning rate scheduler are built] datetime: 2022-11-24 20:23:50 
+0: > building train, validation, and test datasets ...
+0:  > datasets target sizes (minimum size):
+0:     train:      740269
+0:     validation: 768
+0:     test:       256
+0: > building train, validation, and test datasets for GPT ...
+0:  > building dataset index ...
+0:     reading sizes...
+0:     reading pointers...
+0:     reading document index...
+0:     creating numpy buffer of mmap...
+0:     creating memory view of numpy buffer...
+0:  > finished creating indexed dataset in 0.007337 seconds
+0:     number of documents: 210604984
+0:  > dataset split:
+0:     train:
+0:      document indices in [0, 199864130) total of 199864130 documents
+0:     validation:
+0:      document indices in [199864130, 210394379) total of 10530249 documents
+0:     test:
+0:      document indices in [210394379, 210604984) total of 210605 documents
+0:  > WARNING: could not find index map files, building the indices on rank 0 ...
+0:  > only one epoch required, setting separate_last_epoch to False
+0:  > elasped time to build and save doc-idx mapping (seconds): 14.660523
+0:     using:
+0:      number of documents:       199864130
+0:      number of epochs:          1
+0:      sequence length:           2048
+0:      total number of samples:   173377816
+0:  > elasped time to build and save sample-idx mapping (seconds): 4.179427
+0:  > building shuffle index with split [0, 173377816) and [173377816, 173377816) ...
+0:  > elasped time to build and save shuffle-idx mapping (seconds): 10.306571
+0:  > loading doc-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_train_indexmap_740269ns_2048sl_1234s_doc_idx.npy
+0:  > loading sample-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_train_indexmap_740269ns_2048sl_1234s_sample_idx.npy
+0:  > loading shuffle-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_train_indexmap_740269ns_2048sl_1234s_shuffle_idx.npy
+0:     loaded indexed file in 0.094 seconds
+0:     total number of samples: 173377817
+0:     total number of epochs: 1
+0:  > WARNING: could not find index map files, building the indices on rank 0 ...
+0:  > only one epoch required, setting separate_last_epoch to False
+0:  > elasped time to build and save doc-idx mapping (seconds): 0.496899
+0:     using:
+0:      number of documents:       10530249
+0:      number of epochs:          1
+0:      sequence length:           2048
+0:      total number of samples:   9118344
+0:  > elasped time to build and save sample-idx mapping (seconds): 0.216813
+0:  > building shuffle index with split [0, 9118344) and [9118344, 9118344) ...
+0:  > elasped time to build and save shuffle-idx mapping (seconds): 0.265463
+0:  > loading doc-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_valid_indexmap_768ns_2048sl_1234s_doc_idx.npy
+0:  > loading sample-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_valid_indexmap_768ns_2048sl_1234s_sample_idx.npy
+0:  > loading shuffle-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_valid_indexmap_768ns_2048sl_1234s_shuffle_idx.npy
+0:     loaded indexed file in 0.039 seconds
+0:     total number of samples: 9118345
+0:     total number of epochs: 1
+0:  > loading doc-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_test_indexmap_256ns_2048sl_1234s_doc_idx.npy
+0:  > loading sample-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_test_indexmap_256ns_2048sl_1234s_sample_idx.npy
+0:  > loading shuffle-idx mapping from /scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document_test_indexmap_256ns_2048sl_1234s_shuffle_idx.npy
+0:     loaded indexed file in 0.068 seconds
+0:     total number of samples: 182928
+0:     total number of epochs: 1
+0: > finished creating GPT datasets ...
+0: [after dataloaders are built] datetime: 2022-11-24 20:24:35 
+0: done with setup ...
+0: training ...
+0: Number of parameters: [tensor rank - pipeline rank] w/ and w/o embeddings:
+7: time (ms) | model-and-optimizer-setup: 17666.43 | train/valid/test-data-iterators-setup: 45499.83
+0: [000-000] 1.0963B / 1.0025B
+0: [before the start of training step] datetime: 2022-11-24 20:24:36 
+0: [Rank 0] (after 10 iterations) memory (MB) | allocated: 10138.55712890625 | max allocated: 54070.994140625 | reserved: 55702.0 | max reserved: 55702.0
+7:  iteration       10/    2891 | consumed samples:         2560 | consumed tokens:      5242880 | elapsed time per iteration (s): 3.17 | learning rate: 6.916E-05 | global batch size:   256 | lm loss: 9.765865E+00 | grad norm: 2.642 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 80.797 | TFLOPs: 19.55 |
+7:  iteration       20/    2891 | consumed samples:         5120 | consumed tokens:     10485760 | elapsed time per iteration (s): 1.28 | learning rate: 1.383E-04 | global batch size:   256 | lm loss: 8.146842E+00 | grad norm: 2.867 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.251 | TFLOPs: 48.46 |
+7:  iteration       30/    2891 | consumed samples:         7680 | consumed tokens:     15728640 | elapsed time per iteration (s): 1.29 | learning rate: 2.000E-04 | global batch size:   256 | lm loss: 7.337408E+00 | grad norm: 0.904 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.368 | TFLOPs: 48.00 |
+7:  iteration       40/    2891 | consumed samples:        10240 | consumed tokens:     20971520 | elapsed time per iteration (s): 1.29 | learning rate: 2.000E-04 | global batch size:   256 | lm loss: 7.187230E+00 | grad norm: 1.020 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.864 | TFLOPs: 47.88 |
+7:  iteration       50/    2891 | consumed samples:        12800 | consumed tokens:     26214400 | elapsed time per iteration (s): 1.30 | learning rate: 2.000E-04 | global batch size:   256 | lm loss: 6.993211E+00 | grad norm: 1.010 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.599 | TFLOPs: 47.58 |
+7:  iteration       60/    2891 | consumed samples:        15360 | consumed tokens:     31457280 | elapsed time per iteration (s): 1.27 | learning rate: 1.999E-04 | global batch size:   256 | lm loss: 6.864358E+00 | grad norm: 1.450 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.155 | TFLOPs: 48.68 |
+7:  iteration       70/    2891 | consumed samples:        17920 | consumed tokens:     36700160 | elapsed time per iteration (s): 1.33 | learning rate: 1.999E-04 | global batch size:   256 | lm loss: 6.687963E+00 | grad norm: 0.927 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.837 | TFLOPs: 46.42 |
+7:  iteration       80/    2891 | consumed samples:        20480 | consumed tokens:     41943040 | elapsed time per iteration (s): 1.32 | learning rate: 1.999E-04 | global batch size:   256 | lm loss: 6.524078E+00 | grad norm: 1.539 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.935 | TFLOPs: 46.93 |
+7:  iteration       90/    2891 | consumed samples:        23040 | consumed tokens:     47185920 | elapsed time per iteration (s): 1.31 | learning rate: 1.998E-04 | global batch size:   256 | lm loss: 6.442695E+00 | grad norm: 0.947 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.661 | TFLOPs: 47.35 |
+7:  iteration      100/    2891 | consumed samples:        25600 | consumed tokens:     52428800 | elapsed time per iteration (s): 1.34 | learning rate: 1.997E-04 | global batch size:   256 | lm loss: 6.283909E+00 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.825 | TFLOPs: 46.18 |
+7:  iteration      110/    2891 | consumed samples:        28160 | consumed tokens:     57671680 | elapsed time per iteration (s): 1.36 | learning rate: 1.996E-04 | global batch size:   256 | lm loss: 6.177509E+00 | grad norm: 0.893 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 188.043 | TFLOPs: 45.50 |
+7:  iteration      120/    2891 | consumed samples:        30720 | consumed tokens:     62914560 | elapsed time per iteration (s): 1.31 | learning rate: 1.996E-04 | global batch size:   256 | lm loss: 6.081058E+00 | grad norm: 0.769 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.175 | TFLOPs: 47.23 |
+7:  iteration      130/    2891 | consumed samples:        33280 | consumed tokens:     68157440 | elapsed time per iteration (s): 1.29 | learning rate: 1.994E-04 | global batch size:   256 | lm loss: 5.956179E+00 | grad norm: 0.468 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.579 | TFLOPs: 48.05 |
+7:  iteration      140/    2891 | consumed samples:        35840 | consumed tokens:     73400320 | elapsed time per iteration (s): 1.33 | learning rate: 1.993E-04 | global batch size:   256 | lm loss: 5.895684E+00 | grad norm: 1.031 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 192.735 | TFLOPs: 46.64 |
+7:  iteration      150/    2891 | consumed samples:        38400 | consumed tokens:     78643200 | elapsed time per iteration (s): 1.34 | learning rate: 1.992E-04 | global batch size:   256 | lm loss: 5.840030E+00 | grad norm: 0.556 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.486 | TFLOPs: 46.10 |
+7:  iteration      160/    2891 | consumed samples:        40960 | consumed tokens:     83886080 | elapsed time per iteration (s): 1.34 | learning rate: 1.991E-04 | global batch size:   256 | lm loss: 5.732294E+00 | grad norm: 0.774 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.243 | TFLOPs: 46.28 |
+7:  iteration      170/    2891 | consumed samples:        43520 | consumed tokens:     89128960 | elapsed time per iteration (s): 1.28 | learning rate: 1.989E-04 | global batch size:   256 | lm loss: 5.720359E+00 | grad norm: 0.544 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.358 | TFLOPs: 48.24 |
+7:  iteration      180/    2891 | consumed samples:        46080 | consumed tokens:     94371840 | elapsed time per iteration (s): 1.29 | learning rate: 1.988E-04 | global batch size:   256 | lm loss: 5.621735E+00 | grad norm: 0.568 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.849 | TFLOPs: 47.88 |
+7:  iteration      190/    2891 | consumed samples:        48640 | consumed tokens:     99614720 | elapsed time per iteration (s): 1.32 | learning rate: 1.986E-04 | global batch size:   256 | lm loss: 5.592916E+00 | grad norm: 0.628 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.723 | TFLOPs: 46.88 |
+7:  iteration      200/    2891 | consumed samples:        51200 | consumed tokens:    104857600 | elapsed time per iteration (s): 1.34 | learning rate: 1.984E-04 | global batch size:   256 | lm loss: 5.548849E+00 | grad norm: 0.676 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.961 | TFLOPs: 46.21 |
+7:  iteration      210/    2891 | consumed samples:        53760 | consumed tokens:    110100480 | elapsed time per iteration (s): 1.27 | learning rate: 1.982E-04 | global batch size:   256 | lm loss: 5.514834E+00 | grad norm: 0.624 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.962 | TFLOPs: 48.63 |
+7:  iteration      220/    2891 | consumed samples:        56320 | consumed tokens:    115343360 | elapsed time per iteration (s): 1.29 | learning rate: 1.980E-04 | global batch size:   256 | lm loss: 5.464627E+00 | grad norm: 1.075 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.527 | TFLOPs: 48.04 |
+7:  iteration      230/    2891 | consumed samples:        58880 | consumed tokens:    120586240 | elapsed time per iteration (s): 1.29 | learning rate: 1.978E-04 | global batch size:   256 | lm loss: 5.458317E+00 | grad norm: 0.516 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.832 | TFLOPs: 47.87 |
+7:  iteration      240/    2891 | consumed samples:        61440 | consumed tokens:    125829120 | elapsed time per iteration (s): 1.34 | learning rate: 1.976E-04 | global batch size:   256 | lm loss: 5.414064E+00 | grad norm: 0.392 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.782 | TFLOPs: 46.17 |
+7:  iteration      250/    2891 | consumed samples:        64000 | consumed tokens:    131072000 | elapsed time per iteration (s): 1.29 | learning rate: 1.974E-04 | global batch size:   256 | lm loss: 5.348185E+00 | grad norm: 0.478 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.265 | TFLOPs: 47.98 |
+7:  iteration      260/    2891 | consumed samples:        66560 | consumed tokens:    136314880 | elapsed time per iteration (s): 1.31 | learning rate: 1.971E-04 | global batch size:   256 | lm loss: 5.327634E+00 | grad norm: 0.475 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.119 | TFLOPs: 47.46 |
+7:  iteration      270/    2891 | consumed samples:        69120 | consumed tokens:    141557760 | elapsed time per iteration (s): 1.31 | learning rate: 1.969E-04 | global batch size:   256 | lm loss: 5.248993E+00 | grad norm: 0.402 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.331 | TFLOPs: 47.27 |
+7:  iteration      280/    2891 | consumed samples:        71680 | consumed tokens:    146800640 | elapsed time per iteration (s): 1.29 | learning rate: 1.966E-04 | global batch size:   256 | lm loss: 5.236750E+00 | grad norm: 0.702 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.965 | TFLOPs: 47.91 |
+7:  iteration      290/    2891 | consumed samples:        74240 | consumed tokens:    152043520 | elapsed time per iteration (s): 1.27 | learning rate: 1.963E-04 | global batch size:   256 | lm loss: 5.197227E+00 | grad norm: 0.753 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.481 | TFLOPs: 48.76 |
+7:  iteration      300/    2891 | consumed samples:        76800 | consumed tokens:    157286400 | elapsed time per iteration (s): 1.29 | learning rate: 1.960E-04 | global batch size:   256 | lm loss: 5.120818E+00 | grad norm: 0.417 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.938 | TFLOPs: 47.90 |
+7:  iteration      310/    2891 | consumed samples:        79360 | consumed tokens:    162529280 | elapsed time per iteration (s): 1.36 | learning rate: 1.958E-04 | global batch size:   256 | lm loss: 5.137289E+00 | grad norm: 0.910 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 188.596 | TFLOPs: 45.64 |
+7:  iteration      320/    2891 | consumed samples:        81920 | consumed tokens:    167772160 | elapsed time per iteration (s): 1.31 | learning rate: 1.954E-04 | global batch size:   256 | lm loss: 5.131575E+00 | grad norm: 0.554 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.621 | TFLOPs: 47.34 |
+7:  iteration      330/    2891 | consumed samples:        84480 | consumed tokens:    173015040 | elapsed time per iteration (s): 1.30 | learning rate: 1.951E-04 | global batch size:   256 | lm loss: 5.089932E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.550 | TFLOPs: 47.81 |
+7:  iteration      340/    2891 | consumed samples:        87040 | consumed tokens:    178257920 | elapsed time per iteration (s): 1.32 | learning rate: 1.948E-04 | global batch size:   256 | lm loss: 5.049382E+00 | grad norm: 0.433 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.216 | TFLOPs: 46.76 |
+7:  iteration      350/    2891 | consumed samples:        89600 | consumed tokens:    183500800 | elapsed time per iteration (s): 1.30 | learning rate: 1.945E-04 | global batch size:   256 | lm loss: 4.994262E+00 | grad norm: 0.421 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.317 | TFLOPs: 47.75 |
+7:  iteration      360/    2891 | consumed samples:        92160 | consumed tokens:    188743680 | elapsed time per iteration (s): 1.29 | learning rate: 1.941E-04 | global batch size:   256 | lm loss: 4.978718E+00 | grad norm: 0.835 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.947 | TFLOPs: 48.14 |
+7:  iteration      370/    2891 | consumed samples:        94720 | consumed tokens:    193986560 | elapsed time per iteration (s): 1.28 | learning rate: 1.938E-04 | global batch size:   256 | lm loss: 4.963504E+00 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.884 | TFLOPs: 48.37 |
+7:  iteration      380/    2891 | consumed samples:        97280 | consumed tokens:    199229440 | elapsed time per iteration (s): 1.28 | learning rate: 1.934E-04 | global batch size:   256 | lm loss: 4.920103E+00 | grad norm: 0.393 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.998 | TFLOPs: 48.40 |
+7:  iteration      390/    2891 | consumed samples:        99840 | consumed tokens:    204472320 | elapsed time per iteration (s): 1.32 | learning rate: 1.930E-04 | global batch size:   256 | lm loss: 4.859517E+00 | grad norm: 0.816 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.480 | TFLOPs: 46.82 |
+7:  iteration      400/    2891 | consumed samples:       102400 | consumed tokens:    209715200 | elapsed time per iteration (s): 1.28 | learning rate: 1.926E-04 | global batch size:   256 | lm loss: 4.870290E+00 | grad norm: 0.389 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.839 | TFLOPs: 48.36 |
+7:  iteration      410/    2891 | consumed samples:       104960 | consumed tokens:    214958080 | elapsed time per iteration (s): 1.29 | learning rate: 1.922E-04 | global batch size:   256 | lm loss: 4.825578E+00 | grad norm: 0.595 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.841 | TFLOPs: 47.88 |
+7:  iteration      420/    2891 | consumed samples:       107520 | consumed tokens:    220200960 | elapsed time per iteration (s): 1.34 | learning rate: 1.918E-04 | global batch size:   256 | lm loss: 4.905747E+00 | grad norm: 0.545 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.162 | TFLOPs: 46.26 |
+7:  iteration      430/    2891 | consumed samples:       110080 | consumed tokens:    225443840 | elapsed time per iteration (s): 1.32 | learning rate: 1.914E-04 | global batch size:   256 | lm loss: 4.809043E+00 | grad norm: 0.381 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.730 | TFLOPs: 46.88 |
+7:  iteration      440/    2891 | consumed samples:       112640 | consumed tokens:    230686720 | elapsed time per iteration (s): 1.32 | learning rate: 1.910E-04 | global batch size:   256 | lm loss: 4.734610E+00 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.402 | TFLOPs: 46.80 |
+7:  iteration      450/    2891 | consumed samples:       115200 | consumed tokens:    235929600 | elapsed time per iteration (s): 1.30 | learning rate: 1.906E-04 | global batch size:   256 | lm loss: 4.696649E+00 | grad norm: 0.631 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.219 | TFLOPs: 47.72 |
+7:  iteration      460/    2891 | consumed samples:       117760 | consumed tokens:    241172480 | elapsed time per iteration (s): 1.32 | learning rate: 1.901E-04 | global batch size:   256 | lm loss: 4.686860E+00 | grad norm: 0.439 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 194.549 | TFLOPs: 47.08 |
+7:  iteration      470/    2891 | consumed samples:       120320 | consumed tokens:    246415360 | elapsed time per iteration (s): 1.31 | learning rate: 1.897E-04 | global batch size:   256 | lm loss: 4.669160E+00 | grad norm: 0.597 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.936 | TFLOPs: 47.41 |
+7:  iteration      480/    2891 | consumed samples:       122880 | consumed tokens:    251658240 | elapsed time per iteration (s): 1.27 | learning rate: 1.892E-04 | global batch size:   256 | lm loss: 4.641114E+00 | grad norm: 0.576 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.315 | TFLOPs: 48.72 |
+7:  iteration      490/    2891 | consumed samples:       125440 | consumed tokens:    256901120 | elapsed time per iteration (s): 1.33 | learning rate: 1.887E-04 | global batch size:   256 | lm loss: 4.564933E+00 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 192.683 | TFLOPs: 46.63 |
+7:  iteration      500/    2891 | consumed samples:       128000 | consumed tokens:    262144000 | elapsed time per iteration (s): 1.28 | learning rate: 1.882E-04 | global batch size:   256 | lm loss: 4.551287E+00 | grad norm: 0.554 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.108 | TFLOPs: 48.42 |
+7:  iteration      510/    2891 | consumed samples:       130560 | consumed tokens:    267386880 | elapsed time per iteration (s): 1.27 | learning rate: 1.877E-04 | global batch size:   256 | lm loss: 4.540755E+00 | grad norm: 0.559 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.055 | TFLOPs: 48.65 |
+7:  iteration      520/    2891 | consumed samples:       133120 | consumed tokens:    272629760 | elapsed time per iteration (s): 1.31 | learning rate: 1.872E-04 | global batch size:   256 | lm loss: 4.483091E+00 | grad norm: 0.410 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.698 | TFLOPs: 47.36 |
+7:  iteration      530/    2891 | consumed samples:       135680 | consumed tokens:    277872640 | elapsed time per iteration (s): 1.31 | learning rate: 1.867E-04 | global batch size:   256 | lm loss: 4.402271E+00 | grad norm: 0.623 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.908 | TFLOPs: 47.41 |
+7:  iteration      540/    2891 | consumed samples:       138240 | consumed tokens:    283115520 | elapsed time per iteration (s): 1.30 | learning rate: 1.862E-04 | global batch size:   256 | lm loss: 4.453804E+00 | grad norm: 0.509 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.574 | TFLOPs: 47.57 |
+7:  iteration      550/    2891 | consumed samples:       140800 | consumed tokens:    288358400 | elapsed time per iteration (s): 1.30 | learning rate: 1.857E-04 | global batch size:   256 | lm loss: 4.442201E+00 | grad norm: 0.499 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.722 | TFLOPs: 47.60 |
+7:  iteration      560/    2891 | consumed samples:       143360 | consumed tokens:    293601280 | elapsed time per iteration (s): 1.28 | learning rate: 1.851E-04 | global batch size:   256 | lm loss: 4.416363E+00 | grad norm: 0.582 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.477 | TFLOPs: 48.27 |
+7:  iteration      570/    2891 | consumed samples:       145920 | consumed tokens:    298844160 | elapsed time per iteration (s): 1.28 | learning rate: 1.846E-04 | global batch size:   256 | lm loss: 4.346658E+00 | grad norm: 0.564 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.171 | TFLOPs: 48.44 |
+7:  iteration      580/    2891 | consumed samples:       148480 | consumed tokens:    304087040 | elapsed time per iteration (s): 1.29 | learning rate: 1.840E-04 | global batch size:   256 | lm loss: 4.270995E+00 | grad norm: 0.533 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.027 | TFLOPs: 48.16 |
+7:  iteration      590/    2891 | consumed samples:       151040 | consumed tokens:    309329920 | elapsed time per iteration (s): 1.29 | learning rate: 1.835E-04 | global batch size:   256 | lm loss: 4.192557E+00 | grad norm: 0.726 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.289 | TFLOPs: 47.98 |
+7:  iteration      600/    2891 | consumed samples:       153600 | consumed tokens:    314572800 | elapsed time per iteration (s): 1.32 | learning rate: 1.829E-04 | global batch size:   256 | lm loss: 4.170487E+00 | grad norm: 0.641 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.854 | TFLOPs: 46.91 |
+7:  iteration      610/    2891 | consumed samples:       156160 | consumed tokens:    319815680 | elapsed time per iteration (s): 1.29 | learning rate: 1.823E-04 | global batch size:   256 | lm loss: 4.136170E+00 | grad norm: 0.491 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.938 | TFLOPs: 47.90 |
+7:  iteration      620/    2891 | consumed samples:       158720 | consumed tokens:    325058560 | elapsed time per iteration (s): 1.33 | learning rate: 1.817E-04 | global batch size:   256 | lm loss: 4.055492E+00 | grad norm: 0.390 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.799 | TFLOPs: 46.41 |
+7:  iteration      630/    2891 | consumed samples:       161280 | consumed tokens:    330301440 | elapsed time per iteration (s): 1.29 | learning rate: 1.811E-04 | global batch size:   256 | lm loss: 4.104292E+00 | grad norm: 0.508 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.016 | TFLOPs: 48.16 |
+7:  iteration      640/    2891 | consumed samples:       163840 | consumed tokens:    335544320 | elapsed time per iteration (s): 1.27 | learning rate: 1.805E-04 | global batch size:   256 | lm loss: 4.017815E+00 | grad norm: 0.487 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.790 | TFLOPs: 48.59 |
+7:  iteration      650/    2891 | consumed samples:       166400 | consumed tokens:    340787200 | elapsed time per iteration (s): 1.32 | learning rate: 1.799E-04 | global batch size:   256 | lm loss: 3.992767E+00 | grad norm: 0.412 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.918 | TFLOPs: 46.93 |
+7:  iteration      660/    2891 | consumed samples:       168960 | consumed tokens:    346030080 | elapsed time per iteration (s): 1.30 | learning rate: 1.793E-04 | global batch size:   256 | lm loss: 3.913707E+00 | grad norm: 0.443 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.450 | TFLOPs: 47.78 |
+7:  iteration      670/    2891 | consumed samples:       171520 | consumed tokens:    351272960 | elapsed time per iteration (s): 1.30 | learning rate: 1.786E-04 | global batch size:   256 | lm loss: 3.837637E+00 | grad norm: 0.506 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.229 | TFLOPs: 47.73 |
+7:  iteration      680/    2891 | consumed samples:       174080 | consumed tokens:    356515840 | elapsed time per iteration (s): 1.34 | learning rate: 1.780E-04 | global batch size:   256 | lm loss: 3.840733E+00 | grad norm: 0.429 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.224 | TFLOPs: 46.27 |
+7:  iteration      690/    2891 | consumed samples:       176640 | consumed tokens:    361758720 | elapsed time per iteration (s): 1.33 | learning rate: 1.773E-04 | global batch size:   256 | lm loss: 3.826342E+00 | grad norm: 0.525 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 192.659 | TFLOPs: 46.62 |
+7:  iteration      700/    2891 | consumed samples:       179200 | consumed tokens:    367001600 | elapsed time per iteration (s): 1.31 | learning rate: 1.767E-04 | global batch size:   256 | lm loss: 3.802640E+00 | grad norm: 0.386 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.724 | TFLOPs: 47.36 |
+7:  iteration      710/    2891 | consumed samples:       181760 | consumed tokens:    372244480 | elapsed time per iteration (s): 1.27 | learning rate: 1.760E-04 | global batch size:   256 | lm loss: 3.762965E+00 | grad norm: 0.394 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.604 | TFLOPs: 48.79 |
+7:  iteration      720/    2891 | consumed samples:       184320 | consumed tokens:    377487360 | elapsed time per iteration (s): 1.29 | learning rate: 1.753E-04 | global batch size:   256 | lm loss: 3.796334E+00 | grad norm: 0.524 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.396 | TFLOPs: 48.01 |
+7:  iteration      730/    2891 | consumed samples:       186880 | consumed tokens:    382730240 | elapsed time per iteration (s): 1.33 | learning rate: 1.747E-04 | global batch size:   256 | lm loss: 3.754204E+00 | grad norm: 0.469 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 192.067 | TFLOPs: 46.48 |
+7:  iteration      740/    2891 | consumed samples:       189440 | consumed tokens:    387973120 | elapsed time per iteration (s): 1.29 | learning rate: 1.740E-04 | global batch size:   256 | lm loss: 3.693951E+00 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.752 | TFLOPs: 47.85 |
+7:  iteration      750/    2891 | consumed samples:       192000 | consumed tokens:    393216000 | elapsed time per iteration (s): 1.28 | learning rate: 1.733E-04 | global batch size:   256 | lm loss: 3.659017E+00 | grad norm: 0.364 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.446 | TFLOPs: 48.26 |
+7:  iteration      760/    2891 | consumed samples:       194560 | consumed tokens:    398458880 | elapsed time per iteration (s): 1.30 | learning rate: 1.726E-04 | global batch size:   256 | lm loss: 3.682241E+00 | grad norm: 0.333 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.039 | TFLOPs: 47.68 |
+7:  iteration      770/    2891 | consumed samples:       197120 | consumed tokens:    403701760 | elapsed time per iteration (s): 1.29 | learning rate: 1.718E-04 | global batch size:   256 | lm loss: 3.680953E+00 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.135 | TFLOPs: 47.95 |
+7:  iteration      780/    2891 | consumed samples:       199680 | consumed tokens:    408944640 | elapsed time per iteration (s): 1.30 | learning rate: 1.711E-04 | global batch size:   256 | lm loss: 3.615666E+00 | grad norm: 0.419 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.625 | TFLOPs: 47.58 |
+7:  iteration      790/    2891 | consumed samples:       202240 | consumed tokens:    414187520 | elapsed time per iteration (s): 1.31 | learning rate: 1.704E-04 | global batch size:   256 | lm loss: 3.587719E+00 | grad norm: 0.367 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.866 | TFLOPs: 47.40 |
+7:  iteration      800/    2891 | consumed samples:       204800 | consumed tokens:    419430400 | elapsed time per iteration (s): 1.29 | learning rate: 1.697E-04 | global batch size:   256 | lm loss: 3.559205E+00 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.273 | TFLOPs: 47.98 |
+7:  iteration      810/    2891 | consumed samples:       207360 | consumed tokens:    424673280 | elapsed time per iteration (s): 1.31 | learning rate: 1.689E-04 | global batch size:   256 | lm loss: 3.537137E+00 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.035 | TFLOPs: 47.44 |
+7:  iteration      820/    2891 | consumed samples:       209920 | consumed tokens:    429916160 | elapsed time per iteration (s): 1.30 | learning rate: 1.682E-04 | global batch size:   256 | lm loss: 3.544258E+00 | grad norm: 0.284 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.194 | TFLOPs: 47.48 |
+7:  iteration      830/    2891 | consumed samples:       212480 | consumed tokens:    435159040 | elapsed time per iteration (s): 1.29 | learning rate: 1.674E-04 | global batch size:   256 | lm loss: 3.564616E+00 | grad norm: 0.440 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.447 | TFLOPs: 48.02 |
+7:  iteration      840/    2891 | consumed samples:       215040 | consumed tokens:    440401920 | elapsed time per iteration (s): 1.33 | learning rate: 1.666E-04 | global batch size:   256 | lm loss: 3.502379E+00 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.109 | TFLOPs: 46.73 |
+7:  iteration      850/    2891 | consumed samples:       217600 | consumed tokens:    445644800 | elapsed time per iteration (s): 1.31 | learning rate: 1.659E-04 | global batch size:   256 | lm loss: 3.491462E+00 | grad norm: 0.331 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.419 | TFLOPs: 47.29 |
+7:  iteration      860/    2891 | consumed samples:       220160 | consumed tokens:    450887680 | elapsed time per iteration (s): 1.30 | learning rate: 1.651E-04 | global batch size:   256 | lm loss: 3.473122E+00 | grad norm: 0.414 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.474 | TFLOPs: 47.79 |
+7:  iteration      870/    2891 | consumed samples:       222720 | consumed tokens:    456130560 | elapsed time per iteration (s): 1.36 | learning rate: 1.643E-04 | global batch size:   256 | lm loss: 3.498838E+00 | grad norm: 0.408 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 187.592 | TFLOPs: 45.40 |
+7:  iteration      880/    2891 | consumed samples:       225280 | consumed tokens:    461373440 | elapsed time per iteration (s): 1.30 | learning rate: 1.635E-04 | global batch size:   256 | lm loss: 3.505949E+00 | grad norm: 0.668 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.724 | TFLOPs: 47.61 |
+7:  iteration      890/    2891 | consumed samples:       227840 | consumed tokens:    466616320 | elapsed time per iteration (s): 1.35 | learning rate: 1.627E-04 | global batch size:   256 | lm loss: 3.474186E+00 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.163 | TFLOPs: 46.02 |
+7:  iteration      900/    2891 | consumed samples:       230400 | consumed tokens:    471859200 | elapsed time per iteration (s): 1.28 | learning rate: 1.619E-04 | global batch size:   256 | lm loss: 3.431589E+00 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.591 | TFLOPs: 48.54 |
+7:  iteration      910/    2891 | consumed samples:       232960 | consumed tokens:    477102080 | elapsed time per iteration (s): 1.29 | learning rate: 1.611E-04 | global batch size:   256 | lm loss: 3.452892E+00 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.501 | TFLOPs: 48.04 |
+7:  iteration      920/    2891 | consumed samples:       235520 | consumed tokens:    482344960 | elapsed time per iteration (s): 1.31 | learning rate: 1.603E-04 | global batch size:   256 | lm loss: 3.445724E+00 | grad norm: 0.451 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 194.968 | TFLOPs: 47.18 |
+7:  iteration      930/    2891 | consumed samples:       238080 | consumed tokens:    487587840 | elapsed time per iteration (s): 1.31 | learning rate: 1.595E-04 | global batch size:   256 | lm loss: 3.433819E+00 | grad norm: 0.383 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.505 | TFLOPs: 47.31 |
+7:  iteration      940/    2891 | consumed samples:       240640 | consumed tokens:    492830720 | elapsed time per iteration (s): 1.34 | learning rate: 1.586E-04 | global batch size:   256 | lm loss: 3.396906E+00 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 191.149 | TFLOPs: 46.26 |
+7:  iteration      950/    2891 | consumed samples:       243200 | consumed tokens:    498073600 | elapsed time per iteration (s): 1.31 | learning rate: 1.578E-04 | global batch size:   256 | lm loss: 3.402228E+00 | grad norm: 0.337 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 195.917 | TFLOPs: 47.41 |
+7:  iteration      960/    2891 | consumed samples:       245760 | consumed tokens:    503316480 | elapsed time per iteration (s): 1.27 | learning rate: 1.570E-04 | global batch size:   256 | lm loss: 3.365099E+00 | grad norm: 0.277 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.713 | TFLOPs: 48.81 |
+7:  iteration      970/    2891 | consumed samples:       248320 | consumed tokens:    508559360 | elapsed time per iteration (s): 1.32 | learning rate: 1.561E-04 | global batch size:   256 | lm loss: 3.364809E+00 | grad norm: 0.290 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 194.108 | TFLOPs: 46.97 |
+7:  iteration      980/    2891 | consumed samples:       250880 | consumed tokens:    513802240 | elapsed time per iteration (s): 1.28 | learning rate: 1.553E-04 | global batch size:   256 | lm loss: 3.314942E+00 | grad norm: 0.248 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.369 | TFLOPs: 48.49 |
+7:  iteration      990/    2891 | consumed samples:       253440 | consumed tokens:    519045120 | elapsed time per iteration (s): 1.34 | learning rate: 1.544E-04 | global batch size:   256 | lm loss: 3.328453E+00 | grad norm: 0.318 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 190.922 | TFLOPs: 46.20 |
+7:  iteration     1000/    2891 | consumed samples:       256000 | consumed tokens:    524288000 | elapsed time per iteration (s): 1.28 | learning rate: 1.536E-04 | global batch size:   256 | lm loss: 3.317171E+00 | grad norm: 0.328 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.136 | TFLOPs: 48.43 |
+7: ------------------------------------------------------------------------------------------
+7: valid loss at iteration 1000 | lm loss value: 3.278170E+00 | lm loss PPL: 2.652719E+01 | 
+7: ------------------------------------------------------------------------------------------
+0: saving checkpoint at iteration    1000 to checkpoints_1b1
+0: [2022-11-24 20:46:40,054] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is begin to save!
+0: [2022-11-24 20:46:40,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_01-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,422] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_01-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,423] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_03-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_03-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,506] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_04-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_04-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,583] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_05-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,660] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_05-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,661] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_06-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,736] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_06-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_07-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,813] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_07-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,814] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_08-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_08-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,887] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_09-model_00-model_states.pt...
+0: [2022-11-24 20:46:40,964] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_09-model_00-model_states.pt.
+0: [2022-11-24 20:46:40,964] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_10-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_10-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,040] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_11-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,112] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_11-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,112] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_12-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_12-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_13-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_13-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,262] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_14-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,337] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_14-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,338] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_15-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_15-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,414] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_16-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_16-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,486] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_17-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_17-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,562] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_18-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,640] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_18-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_19-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_19-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_20-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_20-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,789] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_21-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,860] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_21-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_22-model_00-model_states.pt...
+0: [2022-11-24 20:46:41,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_22-model_00-model_states.pt.
+0: [2022-11-24 20:46:41,938] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_23-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_23-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,013] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_24-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_24-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,087] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_25-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_25-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,162] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_26-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_26-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,236] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_27-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,311] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_27-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,311] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_28-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_28-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,385] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/layer_30-model_00-model_states.pt...
+0: [2022-11-24 20:46:42,387] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/layer_30-model_00-model_states.pt.
+0: [2022-11-24 20:46:42,388] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: checkpoints_1b1/global_step1000/mp_rank_00_model_states.pt
+0: [2022-11-24 20:46:42,388] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/mp_rank_00_model_states.pt...
+0: [2022-11-24 20:46:42,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/mp_rank_00_model_states.pt.
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt...
+2: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt...
+4: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt...
+1: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt...
+6: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt...
+3: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt...
+7: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt...
+5: [2022-11-24 20:46:42,413] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step1000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt...
+0: [2022-11-24 20:46:42,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,672] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,672] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,673] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,673] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,677] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,677] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,677] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,677] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,677] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,691] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,691] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.
+0: [2022-11-24 20:46:42,691] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:42,691] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,714] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,714] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,717] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,717] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,717] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,717] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,717] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,729] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,729] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,736] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,736] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,736] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,739] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,739] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+2: [2022-11-24 20:46:42,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.
+2: [2022-11-24 20:46:42,771] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
+2: [2022-11-24 20:46:42,771] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:42,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,773] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,773] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,783] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,783] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,787] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,787] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,787] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,798] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,798] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:42,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,815] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,815] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,784] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,784] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,784] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,784] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,784] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,791] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,791] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,824] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,824] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:42,859] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,859] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,859] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,864] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,864] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,864] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,864] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+4: [2022-11-24 20:46:42,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt.
+4: [2022-11-24 20:46:42,865] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
+4: [2022-11-24 20:46:42,865] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,894] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,894] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,905] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,905] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,916] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,916] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,942] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,942] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:42,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:42,949] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:42,949] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,956] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:42,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,957] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,957] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,960] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,960] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,960] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,960] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,963] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,963] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt.
+3: [2022-11-24 20:46:42,975] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,975] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,975] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,975] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+3: [2022-11-24 20:46:42,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:42,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:42,977] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:42,977] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:42,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:42,992] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:42,992] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:43,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:43,004] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:43,004] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:43,009] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:43,009] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:43,009] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,011] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,011] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,011] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,011] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,011] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0: [2022-11-24 20:46:43,012] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+0: [2022-11-24 20:46:43,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,014] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,014] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,014] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,014] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+1: [2022-11-24 20:46:43,015] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.
+1: [2022-11-24 20:46:43,015] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
+1: [2022-11-24 20:46:43,015] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+7: [2022-11-24 20:46:43,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt.
+7: [2022-11-24 20:46:43,050] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt
+7: [2022-11-24 20:46:43,050] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+6: [2022-11-24 20:46:43,059] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt.
+6: [2022-11-24 20:46:43,059] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt
+6: [2022-11-24 20:46:43,059] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+5: [2022-11-24 20:46:43,086] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt.
+5: [2022-11-24 20:46:43,087] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step1000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
+5: [2022-11-24 20:46:43,087] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now!
+0:   successfully saved checkpoint at iteration    1000 to checkpoints_1b1
+7: time (ms) | save-checkpoint: 3040.36
+7:  iteration     1010/    2891 | consumed samples:       258560 | consumed tokens:    529530880 | elapsed time per iteration (s): 1.63 | learning rate: 1.527E-04 | global batch size:   256 | lm loss: 3.343962E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 157.381 | TFLOPs: 38.08 |
+7:  iteration     1020/    2891 | consumed samples:       261120 | consumed tokens:    534773760 | elapsed time per iteration (s): 1.27 | learning rate: 1.518E-04 | global batch size:   256 | lm loss: 3.303916E+00 | grad norm: 0.258 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.840 | TFLOPs: 48.84 |
+7:  iteration     1030/    2891 | consumed samples:       263680 | consumed tokens:    540016640 | elapsed time per iteration (s): 1.27 | learning rate: 1.509E-04 | global batch size:   256 | lm loss: 3.294590E+00 | grad norm: 0.280 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.908 | TFLOPs: 48.86 |
+7:  iteration     1040/    2891 | consumed samples:       266240 | consumed tokens:    545259520 | elapsed time per iteration (s): 1.28 | learning rate: 1.501E-04 | global batch size:   256 | lm loss: 3.344618E+00 | grad norm: 0.379 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.636 | TFLOPs: 48.31 |
+7:  iteration     1050/    2891 | consumed samples:       268800 | consumed tokens:    550502400 | elapsed time per iteration (s): 1.27 | learning rate: 1.492E-04 | global batch size:   256 | lm loss: 3.318086E+00 | grad norm: 0.359 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.480 | TFLOPs: 48.76 |
+7:  iteration     1060/    2891 | consumed samples:       271360 | consumed tokens:    555745280 | elapsed time per iteration (s): 1.26 | learning rate: 1.483E-04 | global batch size:   256 | lm loss: 3.272342E+00 | grad norm: 0.287 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.079 | TFLOPs: 49.14 |
+7:  iteration     1070/    2891 | consumed samples:       273920 | consumed tokens:    560988160 | elapsed time per iteration (s): 1.28 | learning rate: 1.474E-04 | global batch size:   256 | lm loss: 3.262115E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.291 | TFLOPs: 48.47 |
+7:  iteration     1080/    2891 | consumed samples:       276480 | consumed tokens:    566231040 | elapsed time per iteration (s): 1.30 | learning rate: 1.465E-04 | global batch size:   256 | lm loss: 3.210053E+00 | grad norm: 0.319 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.457 | TFLOPs: 47.78 |
+7:  iteration     1090/    2891 | consumed samples:       279040 | consumed tokens:    571473920 | elapsed time per iteration (s): 1.27 | learning rate: 1.456E-04 | global batch size:   256 | lm loss: 3.242656E+00 | grad norm: 0.336 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.319 | TFLOPs: 48.96 |
+7:  iteration     1100/    2891 | consumed samples:       281600 | consumed tokens:    576716800 | elapsed time per iteration (s): 1.27 | learning rate: 1.447E-04 | global batch size:   256 | lm loss: 3.271250E+00 | grad norm: 0.298 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.193 | TFLOPs: 48.93 |
+7:  iteration     1110/    2891 | consumed samples:       284160 | consumed tokens:    581959680 | elapsed time per iteration (s): 1.26 | learning rate: 1.438E-04 | global batch size:   256 | lm loss: 3.224141E+00 | grad norm: 0.274 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.620 | TFLOPs: 49.03 |
+7:  iteration     1120/    2891 | consumed samples:       286720 | consumed tokens:    587202560 | elapsed time per iteration (s): 1.27 | learning rate: 1.428E-04 | global batch size:   256 | lm loss: 3.251974E+00 | grad norm: 0.297 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.598 | TFLOPs: 48.78 |
+7:  iteration     1130/    2891 | consumed samples:       289280 | consumed tokens:    592445440 | elapsed time per iteration (s): 1.29 | learning rate: 1.419E-04 | global batch size:   256 | lm loss: 3.254908E+00 | grad norm: 0.368 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.146 | TFLOPs: 48.19 |
+7:  iteration     1140/    2891 | consumed samples:       291840 | consumed tokens:    597688320 | elapsed time per iteration (s): 1.28 | learning rate: 1.410E-04 | global batch size:   256 | lm loss: 3.207593E+00 | grad norm: 0.268 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.753 | TFLOPs: 48.34 |
+7:  iteration     1150/    2891 | consumed samples:       294400 | consumed tokens:    602931200 | elapsed time per iteration (s): 1.28 | learning rate: 1.401E-04 | global batch size:   256 | lm loss: 3.237651E+00 | grad norm: 0.346 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.528 | TFLOPs: 48.53 |
+7:  iteration     1160/    2891 | consumed samples:       296960 | consumed tokens:    608174080 | elapsed time per iteration (s): 1.27 | learning rate: 1.391E-04 | global batch size:   256 | lm loss: 3.218523E+00 | grad norm: 0.305 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.106 | TFLOPs: 48.91 |
+7:  iteration     1170/    2891 | consumed samples:       299520 | consumed tokens:    613416960 | elapsed time per iteration (s): 1.32 | learning rate: 1.382E-04 | global batch size:   256 | lm loss: 3.215828E+00 | grad norm: 0.308 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 194.426 | TFLOPs: 47.05 |
+7:  iteration     1180/    2891 | consumed samples:       302080 | consumed tokens:    618659840 | elapsed time per iteration (s): 1.28 | learning rate: 1.372E-04 | global batch size:   256 | lm loss: 3.227235E+00 | grad norm: 0.312 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.637 | TFLOPs: 48.55 |
+7:  iteration     1190/    2891 | consumed samples:       304640 | consumed tokens:    623902720 | elapsed time per iteration (s): 1.28 | learning rate: 1.363E-04 | global batch size:   256 | lm loss: 3.230080E+00 | grad norm: 0.278 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.825 | TFLOPs: 48.36 |
+7:  iteration     1200/    2891 | consumed samples:       307200 | consumed tokens:    629145600 | elapsed time per iteration (s): 1.27 | learning rate: 1.354E-04 | global batch size:   256 | lm loss: 3.197777E+00 | grad norm: 0.296 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.552 | TFLOPs: 48.77 |
+7:  iteration     1210/    2891 | consumed samples:       309760 | consumed tokens:    634388480 | elapsed time per iteration (s): 1.29 | learning rate: 1.344E-04 | global batch size:   256 | lm loss: 3.187337E+00 | grad norm: 0.327 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.459 | TFLOPs: 48.03 |
+7:  iteration     1220/    2891 | consumed samples:       312320 | consumed tokens:    639631360 | elapsed time per iteration (s): 1.29 | learning rate: 1.335E-04 | global batch size:   256 | lm loss: 3.127047E+00 | grad norm: 0.288 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.594 | TFLOPs: 48.06 |
+7:  iteration     1230/    2891 | consumed samples:       314880 | consumed tokens:    644874240 | elapsed time per iteration (s): 1.28 | learning rate: 1.325E-04 | global batch size:   256 | lm loss: 3.181544E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.105 | TFLOPs: 48.42 |
+7:  iteration     1240/    2891 | consumed samples:       317440 | consumed tokens:    650117120 | elapsed time per iteration (s): 1.30 | learning rate: 1.315E-04 | global batch size:   256 | lm loss: 3.154024E+00 | grad norm: 0.280 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.355 | TFLOPs: 47.76 |
+7:  iteration     1250/    2891 | consumed samples:       320000 | consumed tokens:    655360000 | elapsed time per iteration (s): 1.29 | learning rate: 1.306E-04 | global batch size:   256 | lm loss: 3.156109E+00 | grad norm: 0.305 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.078 | TFLOPs: 47.93 |
+7:  iteration     1260/    2891 | consumed samples:       322560 | consumed tokens:    660602880 | elapsed time per iteration (s): 1.27 | learning rate: 1.296E-04 | global batch size:   256 | lm loss: 3.096645E+00 | grad norm: 0.323 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.780 | TFLOPs: 48.83 |
+7:  iteration     1270/    2891 | consumed samples:       325120 | consumed tokens:    665845760 | elapsed time per iteration (s): 1.28 | learning rate: 1.287E-04 | global batch size:   256 | lm loss: 3.159548E+00 | grad norm: 0.288 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.403 | TFLOPs: 48.25 |
+7:  iteration     1280/    2891 | consumed samples:       327680 | consumed tokens:    671088640 | elapsed time per iteration (s): 1.29 | learning rate: 1.277E-04 | global batch size:   256 | lm loss: 3.158171E+00 | grad norm: 0.341 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.171 | TFLOPs: 48.20 |
+7:  iteration     1290/    2891 | consumed samples:       330240 | consumed tokens:    676331520 | elapsed time per iteration (s): 1.27 | learning rate: 1.267E-04 | global batch size:   256 | lm loss: 3.041705E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.272 | TFLOPs: 48.71 |
+7:  iteration     1300/    2891 | consumed samples:       332800 | consumed tokens:    681574400 | elapsed time per iteration (s): 1.28 | learning rate: 1.258E-04 | global batch size:   256 | lm loss: 3.118959E+00 | grad norm: 0.324 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.440 | TFLOPs: 48.50 |
+7:  iteration     1310/    2891 | consumed samples:       335360 | consumed tokens:    686817280 | elapsed time per iteration (s): 1.27 | learning rate: 1.248E-04 | global batch size:   256 | lm loss: 3.098573E+00 | grad norm: 0.285 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.799 | TFLOPs: 48.59 |
+7:  iteration     1320/    2891 | consumed samples:       337920 | consumed tokens:    692060160 | elapsed time per iteration (s): 1.27 | learning rate: 1.238E-04 | global batch size:   256 | lm loss: 3.096959E+00 | grad norm: 0.352 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.273 | TFLOPs: 48.71 |
+7:  iteration     1330/    2891 | consumed samples:       340480 | consumed tokens:    697303040 | elapsed time per iteration (s): 1.28 | learning rate: 1.228E-04 | global batch size:   256 | lm loss: 3.100436E+00 | grad norm: 0.271 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.779 | TFLOPs: 48.34 |
+7:  iteration     1340/    2891 | consumed samples:       343040 | consumed tokens:    702545920 | elapsed time per iteration (s): 1.27 | learning rate: 1.218E-04 | global batch size:   256 | lm loss: 3.109594E+00 | grad norm: 0.309 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.820 | TFLOPs: 48.84 |
+7:  iteration     1350/    2891 | consumed samples:       345600 | consumed tokens:    707788800 | elapsed time per iteration (s): 1.27 | learning rate: 1.209E-04 | global batch size:   256 | lm loss: 3.084716E+00 | grad norm: 0.256 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.204 | TFLOPs: 48.69 |
+7:  iteration     1360/    2891 | consumed samples:       348160 | consumed tokens:    713031680 | elapsed time per iteration (s): 1.29 | learning rate: 1.199E-04 | global batch size:   256 | lm loss: 3.048441E+00 | grad norm: 0.309 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.193 | TFLOPs: 47.96 |
+7:  iteration     1370/    2891 | consumed samples:       350720 | consumed tokens:    718274560 | elapsed time per iteration (s): 1.28 | learning rate: 1.189E-04 | global batch size:   256 | lm loss: 3.085259E+00 | grad norm: 0.324 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.007 | TFLOPs: 48.40 |
+7:  iteration     1380/    2891 | consumed samples:       353280 | consumed tokens:    723517440 | elapsed time per iteration (s): 1.29 | learning rate: 1.179E-04 | global batch size:   256 | lm loss: 3.080657E+00 | grad norm: 0.315 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.075 | TFLOPs: 47.93 |
+7:  iteration     1390/    2891 | consumed samples:       355840 | consumed tokens:    728760320 | elapsed time per iteration (s): 1.29 | learning rate: 1.169E-04 | global batch size:   256 | lm loss: 3.076471E+00 | grad norm: 0.294 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.299 | TFLOPs: 47.99 |
+7:  iteration     1400/    2891 | consumed samples:       358400 | consumed tokens:    734003200 | elapsed time per iteration (s): 1.27 | learning rate: 1.160E-04 | global batch size:   256 | lm loss: 3.115252E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.332 | TFLOPs: 48.72 |
+7:  iteration     1410/    2891 | consumed samples:       360960 | consumed tokens:    739246080 | elapsed time per iteration (s): 1.27 | learning rate: 1.150E-04 | global batch size:   256 | lm loss: 3.080489E+00 | grad norm: 0.280 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.687 | TFLOPs: 48.81 |
+7:  iteration     1420/    2891 | consumed samples:       363520 | consumed tokens:    744488960 | elapsed time per iteration (s): 1.28 | learning rate: 1.140E-04 | global batch size:   256 | lm loss: 3.029892E+00 | grad norm: 0.269 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.737 | TFLOPs: 48.33 |
+7:  iteration     1430/    2891 | consumed samples:       366080 | consumed tokens:    749731840 | elapsed time per iteration (s): 1.27 | learning rate: 1.130E-04 | global batch size:   256 | lm loss: 3.031248E+00 | grad norm: 0.285 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.616 | TFLOPs: 48.79 |
+7:  iteration     1440/    2891 | consumed samples:       368640 | consumed tokens:    754974720 | elapsed time per iteration (s): 1.26 | learning rate: 1.120E-04 | global batch size:   256 | lm loss: 3.061755E+00 | grad norm: 0.275 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.804 | TFLOPs: 49.08 |
+7:  iteration     1450/    2891 | consumed samples:       371200 | consumed tokens:    760217600 | elapsed time per iteration (s): 1.29 | learning rate: 1.110E-04 | global batch size:   256 | lm loss: 3.049806E+00 | grad norm: 0.267 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.081 | TFLOPs: 48.18 |
+7:  iteration     1460/    2891 | consumed samples:       373760 | consumed tokens:    765460480 | elapsed time per iteration (s): 1.28 | learning rate: 1.100E-04 | global batch size:   256 | lm loss: 3.061333E+00 | grad norm: 0.270 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.315 | TFLOPs: 48.47 |
+7:  iteration     1470/    2891 | consumed samples:       376320 | consumed tokens:    770703360 | elapsed time per iteration (s): 1.29 | learning rate: 1.090E-04 | global batch size:   256 | lm loss: 3.027428E+00 | grad norm: 0.283 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.854 | TFLOPs: 48.12 |
+7:  iteration     1480/    2891 | consumed samples:       378880 | consumed tokens:    775946240 | elapsed time per iteration (s): 1.27 | learning rate: 1.081E-04 | global batch size:   256 | lm loss: 3.047362E+00 | grad norm: 0.278 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.073 | TFLOPs: 48.90 |
+7:  iteration     1490/    2891 | consumed samples:       381440 | consumed tokens:    781189120 | elapsed time per iteration (s): 1.29 | learning rate: 1.071E-04 | global batch size:   256 | lm loss: 3.060497E+00 | grad norm: 0.263 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.724 | TFLOPs: 47.85 |
+7:  iteration     1500/    2891 | consumed samples:       384000 | consumed tokens:    786432000 | elapsed time per iteration (s): 1.29 | learning rate: 1.061E-04 | global batch size:   256 | lm loss: 3.034048E+00 | grad norm: 0.288 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.203 | TFLOPs: 48.21 |
+7:  iteration     1510/    2891 | consumed samples:       386560 | consumed tokens:    791674880 | elapsed time per iteration (s): 1.28 | learning rate: 1.051E-04 | global batch size:   256 | lm loss: 3.035198E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.262 | TFLOPs: 48.22 |
+7:  iteration     1520/    2891 | consumed samples:       389120 | consumed tokens:    796917760 | elapsed time per iteration (s): 1.27 | learning rate: 1.041E-04 | global batch size:   256 | lm loss: 3.031193E+00 | grad norm: 0.308 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.156 | TFLOPs: 48.68 |
+7:  iteration     1530/    2891 | consumed samples:       391680 | consumed tokens:    802160640 | elapsed time per iteration (s): 1.30 | learning rate: 1.031E-04 | global batch size:   256 | lm loss: 3.019461E+00 | grad norm: 0.285 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.566 | TFLOPs: 47.57 |
+7:  iteration     1540/    2891 | consumed samples:       394240 | consumed tokens:    807403520 | elapsed time per iteration (s): 1.29 | learning rate: 1.021E-04 | global batch size:   256 | lm loss: 3.035143E+00 | grad norm: 0.282 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.773 | TFLOPs: 48.10 |
+7:  iteration     1550/    2891 | consumed samples:       396800 | consumed tokens:    812646400 | elapsed time per iteration (s): 1.28 | learning rate: 1.012E-04 | global batch size:   256 | lm loss: 3.008830E+00 | grad norm: 0.279 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.600 | TFLOPs: 48.30 |
+7:  iteration     1560/    2891 | consumed samples:       399360 | consumed tokens:    817889280 | elapsed time per iteration (s): 1.27 | learning rate: 1.002E-04 | global batch size:   256 | lm loss: 2.989631E+00 | grad norm: 0.283 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.113 | TFLOPs: 48.91 |
+7:  iteration     1570/    2891 | consumed samples:       401920 | consumed tokens:    823132160 | elapsed time per iteration (s): 1.26 | learning rate: 9.919E-05 | global batch size:   256 | lm loss: 2.980611E+00 | grad norm: 0.266 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.434 | TFLOPs: 48.99 |
+7:  iteration     1580/    2891 | consumed samples:       404480 | consumed tokens:    828375040 | elapsed time per iteration (s): 1.26 | learning rate: 9.821E-05 | global batch size:   256 | lm loss: 3.008602E+00 | grad norm: 0.259 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.598 | TFLOPs: 49.03 |
+7:  iteration     1590/    2891 | consumed samples:       407040 | consumed tokens:    833617920 | elapsed time per iteration (s): 1.26 | learning rate: 9.723E-05 | global batch size:   256 | lm loss: 3.012232E+00 | grad norm: 0.267 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.457 | TFLOPs: 48.99 |
+7:  iteration     1600/    2891 | consumed samples:       409600 | consumed tokens:    838860800 | elapsed time per iteration (s): 1.29 | learning rate: 9.626E-05 | global batch size:   256 | lm loss: 2.981424E+00 | grad norm: 0.259 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.103 | TFLOPs: 48.18 |
+7:  iteration     1610/    2891 | consumed samples:       412160 | consumed tokens:    844103680 | elapsed time per iteration (s): 1.27 | learning rate: 9.528E-05 | global batch size:   256 | lm loss: 2.965554E+00 | grad norm: 0.291 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.943 | TFLOPs: 48.87 |
+7:  iteration     1620/    2891 | consumed samples:       414720 | consumed tokens:    849346560 | elapsed time per iteration (s): 1.27 | learning rate: 9.431E-05 | global batch size:   256 | lm loss: 2.965051E+00 | grad norm: 0.270 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.491 | TFLOPs: 48.76 |
+7:  iteration     1630/    2891 | consumed samples:       417280 | consumed tokens:    854589440 | elapsed time per iteration (s): 1.28 | learning rate: 9.334E-05 | global batch size:   256 | lm loss: 2.978099E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.896 | TFLOPs: 48.37 |
+7:  iteration     1640/    2891 | consumed samples:       419840 | consumed tokens:    859832320 | elapsed time per iteration (s): 1.27 | learning rate: 9.237E-05 | global batch size:   256 | lm loss: 2.953215E+00 | grad norm: 0.277 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.827 | TFLOPs: 48.84 |
+7:  iteration     1650/    2891 | consumed samples:       422400 | consumed tokens:    865075200 | elapsed time per iteration (s): 1.28 | learning rate: 9.140E-05 | global batch size:   256 | lm loss: 3.011898E+00 | grad norm: 0.271 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.743 | TFLOPs: 48.58 |
+7:  iteration     1660/    2891 | consumed samples:       424960 | consumed tokens:    870318080 | elapsed time per iteration (s): 1.28 | learning rate: 9.043E-05 | global batch size:   256 | lm loss: 2.940250E+00 | grad norm: 0.292 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.596 | TFLOPs: 48.30 |
+7:  iteration     1670/    2891 | consumed samples:       427520 | consumed tokens:    875560960 | elapsed time per iteration (s): 1.27 | learning rate: 8.947E-05 | global batch size:   256 | lm loss: 2.973220E+00 | grad norm: 0.253 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.531 | TFLOPs: 48.77 |
+7:  iteration     1680/    2891 | consumed samples:       430080 | consumed tokens:    880803840 | elapsed time per iteration (s): 1.29 | learning rate: 8.851E-05 | global batch size:   256 | lm loss: 2.948690E+00 | grad norm: 0.283 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.792 | TFLOPs: 48.11 |
+7:  iteration     1690/    2891 | consumed samples:       432640 | consumed tokens:    886046720 | elapsed time per iteration (s): 1.28 | learning rate: 8.755E-05 | global batch size:   256 | lm loss: 2.996978E+00 | grad norm: 0.279 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.272 | TFLOPs: 48.46 |
+7:  iteration     1700/    2891 | consumed samples:       435200 | consumed tokens:    891289600 | elapsed time per iteration (s): 1.26 | learning rate: 8.660E-05 | global batch size:   256 | lm loss: 2.990174E+00 | grad norm: 0.277 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.748 | TFLOPs: 49.06 |
+7:  iteration     1710/    2891 | consumed samples:       437760 | consumed tokens:    896532480 | elapsed time per iteration (s): 1.28 | learning rate: 8.565E-05 | global batch size:   256 | lm loss: 2.959204E+00 | grad norm: 0.292 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.549 | TFLOPs: 48.29 |
+7:  iteration     1720/    2891 | consumed samples:       440320 | consumed tokens:    901775360 | elapsed time per iteration (s): 1.28 | learning rate: 8.470E-05 | global batch size:   256 | lm loss: 2.953564E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.448 | TFLOPs: 48.26 |
+7:  iteration     1730/    2891 | consumed samples:       442880 | consumed tokens:    907018240 | elapsed time per iteration (s): 1.27 | learning rate: 8.375E-05 | global batch size:   256 | lm loss: 2.944229E+00 | grad norm: 0.267 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.498 | TFLOPs: 48.76 |
+7:  iteration     1740/    2891 | consumed samples:       445440 | consumed tokens:    912261120 | elapsed time per iteration (s): 1.27 | learning rate: 8.281E-05 | global batch size:   256 | lm loss: 2.979383E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.580 | TFLOPs: 48.78 |
+7:  iteration     1750/    2891 | consumed samples:       448000 | consumed tokens:    917504000 | elapsed time per iteration (s): 1.27 | learning rate: 8.187E-05 | global batch size:   256 | lm loss: 2.953689E+00 | grad norm: 0.283 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.279 | TFLOPs: 48.71 |
+7:  iteration     1760/    2891 | consumed samples:       450560 | consumed tokens:    922746880 | elapsed time per iteration (s): 1.29 | learning rate: 8.093E-05 | global batch size:   256 | lm loss: 2.963270E+00 | grad norm: 0.284 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.123 | TFLOPs: 48.19 |
+7:  iteration     1770/    2891 | consumed samples:       453120 | consumed tokens:    927989760 | elapsed time per iteration (s): 1.27 | learning rate: 8.000E-05 | global batch size:   256 | lm loss: 2.926622E+00 | grad norm: 0.286 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.467 | TFLOPs: 48.75 |
+7:  iteration     1780/    2891 | consumed samples:       455680 | consumed tokens:    933232640 | elapsed time per iteration (s): 1.28 | learning rate: 7.907E-05 | global batch size:   256 | lm loss: 2.916380E+00 | grad norm: 0.265 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.919 | TFLOPs: 48.38 |
+7:  iteration     1790/    2891 | consumed samples:       458240 | consumed tokens:    938475520 | elapsed time per iteration (s): 1.27 | learning rate: 7.814E-05 | global batch size:   256 | lm loss: 2.971655E+00 | grad norm: 0.278 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.033 | TFLOPs: 48.65 |
+7:  iteration     1800/    2891 | consumed samples:       460800 | consumed tokens:    943718400 | elapsed time per iteration (s): 1.27 | learning rate: 7.722E-05 | global batch size:   256 | lm loss: 2.912868E+00 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.188 | TFLOPs: 48.69 |
+7:  iteration     1810/    2891 | consumed samples:       463360 | consumed tokens:    948961280 | elapsed time per iteration (s): 1.28 | learning rate: 7.630E-05 | global batch size:   256 | lm loss: 2.943017E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.544 | TFLOPs: 48.53 |
+7:  iteration     1820/    2891 | consumed samples:       465920 | consumed tokens:    954204160 | elapsed time per iteration (s): 1.27 | learning rate: 7.539E-05 | global batch size:   256 | lm loss: 2.924209E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.329 | TFLOPs: 48.96 |
+7:  iteration     1830/    2891 | consumed samples:       468480 | consumed tokens:    959447040 | elapsed time per iteration (s): 1.28 | learning rate: 7.448E-05 | global batch size:   256 | lm loss: 2.913163E+00 | grad norm: 0.287 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.035 | TFLOPs: 48.41 |
+7:  iteration     1840/    2891 | consumed samples:       471040 | consumed tokens:    964689920 | elapsed time per iteration (s): 1.29 | learning rate: 7.357E-05 | global batch size:   256 | lm loss: 2.913443E+00 | grad norm: 0.272 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.220 | TFLOPs: 47.97 |
+7:  iteration     1850/    2891 | consumed samples:       473600 | consumed tokens:    969932800 | elapsed time per iteration (s): 1.29 | learning rate: 7.267E-05 | global batch size:   256 | lm loss: 2.921156E+00 | grad norm: 0.270 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.062 | TFLOPs: 47.93 |
+7:  iteration     1860/    2891 | consumed samples:       476160 | consumed tokens:    975175680 | elapsed time per iteration (s): 1.28 | learning rate: 7.178E-05 | global batch size:   256 | lm loss: 2.882126E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.919 | TFLOPs: 48.38 |
+7:  iteration     1870/    2891 | consumed samples:       478720 | consumed tokens:    980418560 | elapsed time per iteration (s): 1.29 | learning rate: 7.088E-05 | global batch size:   256 | lm loss: 2.893600E+00 | grad norm: 0.269 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.662 | TFLOPs: 48.07 |
+7:  iteration     1880/    2891 | consumed samples:       481280 | consumed tokens:    985661440 | elapsed time per iteration (s): 1.27 | learning rate: 7.000E-05 | global batch size:   256 | lm loss: 2.874181E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.771 | TFLOPs: 48.83 |
+7:  iteration     1890/    2891 | consumed samples:       483840 | consumed tokens:    990904320 | elapsed time per iteration (s): 1.28 | learning rate: 6.912E-05 | global batch size:   256 | lm loss: 2.923892E+00 | grad norm: 0.305 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.665 | TFLOPs: 48.56 |
+7:  iteration     1900/    2891 | consumed samples:       486400 | consumed tokens:    996147200 | elapsed time per iteration (s): 1.28 | learning rate: 6.824E-05 | global batch size:   256 | lm loss: 2.861729E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.733 | TFLOPs: 48.58 |
+7:  iteration     1910/    2891 | consumed samples:       488960 | consumed tokens:   1001390080 | elapsed time per iteration (s): 1.27 | learning rate: 6.737E-05 | global batch size:   256 | lm loss: 2.917368E+00 | grad norm: 0.247 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.737 | TFLOPs: 48.82 |
+7:  iteration     1920/    2891 | consumed samples:       491520 | consumed tokens:   1006632960 | elapsed time per iteration (s): 1.27 | learning rate: 6.650E-05 | global batch size:   256 | lm loss: 2.881069E+00 | grad norm: 0.273 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.453 | TFLOPs: 48.75 |
+7:  iteration     1930/    2891 | consumed samples:       494080 | consumed tokens:   1011875840 | elapsed time per iteration (s): 1.27 | learning rate: 6.564E-05 | global batch size:   256 | lm loss: 2.894638E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.454 | TFLOPs: 48.75 |
+7:  iteration     1940/    2891 | consumed samples:       496640 | consumed tokens:   1017118720 | elapsed time per iteration (s): 1.26 | learning rate: 6.478E-05 | global batch size:   256 | lm loss: 2.885247E+00 | grad norm: 0.288 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.584 | TFLOPs: 49.27 |
+7:  iteration     1950/    2891 | consumed samples:       499200 | consumed tokens:   1022361600 | elapsed time per iteration (s): 1.26 | learning rate: 6.393E-05 | global batch size:   256 | lm loss: 2.863762E+00 | grad norm: 0.268 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.102 | TFLOPs: 49.15 |
+7:  iteration     1960/    2891 | consumed samples:       501760 | consumed tokens:   1027604480 | elapsed time per iteration (s): 1.29 | learning rate: 6.308E-05 | global batch size:   256 | lm loss: 2.919002E+00 | grad norm: 0.244 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.463 | TFLOPs: 48.03 |
+7:  iteration     1970/    2891 | consumed samples:       504320 | consumed tokens:   1032847360 | elapsed time per iteration (s): 1.28 | learning rate: 6.224E-05 | global batch size:   256 | lm loss: 2.918430E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.142 | TFLOPs: 48.43 |
+7:  iteration     1980/    2891 | consumed samples:       506880 | consumed tokens:   1038090240 | elapsed time per iteration (s): 1.26 | learning rate: 6.141E-05 | global batch size:   256 | lm loss: 2.889674E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.387 | TFLOPs: 48.98 |
+7:  iteration     1990/    2891 | consumed samples:       509440 | consumed tokens:   1043333120 | elapsed time per iteration (s): 1.26 | learning rate: 6.058E-05 | global batch size:   256 | lm loss: 2.917491E+00 | grad norm: 0.262 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.177 | TFLOPs: 49.17 |
+0: [2022-11-24 21:08:00,155] [INFO] [logging.py:68:log_dist] [Rank 0] step=2000, skipped=0, lr=[5.9757828883278194e-05, 5.9757828883278194e-05, 5.9757828883278194e-05], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)]
+7:  iteration     2000/    2891 | consumed samples:       512000 | consumed tokens:   1048576000 | elapsed time per iteration (s): 1.27 | learning rate: 5.976E-05 | global batch size:   256 | lm loss: 2.907642E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.935 | TFLOPs: 48.62 |
+0: steps: 2000 loss: 2.9851 iter time (s): 1.297 samples/sec: 197.319
+7: ------------------------------------------------------------------------------------------
+7: valid loss at iteration 2000 | lm loss value: 2.796879E+00 | lm loss PPL: 1.639341E+01 | 
+7: ------------------------------------------------------------------------------------------
+0: saving checkpoint at iteration    2000 to checkpoints_1b1
+0: [2022-11-24 21:08:00,577] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2000 is begin to save!
+0: [2022-11-24 21:08:00,581] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_01-model_00-model_states.pt...
+0: [2022-11-24 21:08:00,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_01-model_00-model_states.pt.
+0: [2022-11-24 21:08:00,773] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_03-model_00-model_states.pt...
+0: [2022-11-24 21:08:00,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_03-model_00-model_states.pt.
+0: [2022-11-24 21:08:00,852] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_04-model_00-model_states.pt...
+0: [2022-11-24 21:08:00,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_04-model_00-model_states.pt.
+0: [2022-11-24 21:08:00,926] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_05-model_00-model_states.pt...
+0: [2022-11-24 21:08:00,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_05-model_00-model_states.pt.
+0: [2022-11-24 21:08:00,997] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_06-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_06-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,072] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_07-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,144] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_07-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_08-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_08-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,220] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_09-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_09-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,294] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_10-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,366] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_10-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,367] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_11-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,442] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_11-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,443] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_12-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_12-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,518] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_13-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,591] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_13-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,592] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_14-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_14-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_15-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_15-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_16-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,813] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_16-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,814] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_17-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_17-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,888] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_18-model_00-model_states.pt...
+0: [2022-11-24 21:08:01,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_18-model_00-model_states.pt.
+0: [2022-11-24 21:08:01,960] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_19-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_19-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,038] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_20-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,111] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_20-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,112] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_21-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_21-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_22-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_22-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,261] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_23-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_23-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,332] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_24-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_24-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,410] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_25-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_25-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,483] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_26-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,560] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_26-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,561] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_27-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_27-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,634] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_28-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_28-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,710] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/layer_30-model_00-model_states.pt...
+0: [2022-11-24 21:08:02,711] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/layer_30-model_00-model_states.pt.
+0: [2022-11-24 21:08:02,712] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: checkpoints_1b1/global_step2000/mp_rank_00_model_states.pt
+0: [2022-11-24 21:08:02,712] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/mp_rank_00_model_states.pt...
+0: [2022-11-24 21:08:02,716] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/mp_rank_00_model_states.pt.
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:08:02,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:02,978] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:02,978] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:02,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:02,980] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:02,980] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:02,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:02,981] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:02,981] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:02,985] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:02,985] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:02,985] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:02,985] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:02,986] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:02,986] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:02,976] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:02,977] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:02,977] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:02,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:02,990] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:02,990] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:02,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:02,990] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:02,990] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:02,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:02,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:02,994] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:02,994] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:02,994] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:02,994] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:02,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:02,998] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:02,998] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:02,999] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:02,999] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:02,999] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:03,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:03,002] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:03,002] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,003] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,003] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,003] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,003] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,003] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,004] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,004] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,005] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,005] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,005] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,005] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,005] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:03,007] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,007] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:03,007] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:03,007] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:03,008] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:03,008] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:03,008] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:03,010] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:03,010] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:03,010] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,011] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,011] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,012] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+1: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:08:03,012] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,012] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:03,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:03,013] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:03,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:03,013] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:03,013] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:03,013] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,015] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,015] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,015] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:03,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:03,016] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:03,016] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+3: [2022-11-24 21:08:03,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:08:03,016] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:08:03,016] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+2: [2022-11-24 21:08:03,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:08:03,018] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:08:03,018] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,024] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,024] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+4: [2022-11-24 21:08:03,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:08:03,025] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:08:03,025] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,047] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,047] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,047] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,048] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,048] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,050] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:08:03,050] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,050] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,050] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,054] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,054] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,054] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,054] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,076] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,076] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,078] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,078] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,079] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0: [2022-11-24 21:08:03,080] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:08:03,080] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,081] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,081] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,081] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,088] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,088] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,089] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,089] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,110] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,110] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,114] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,114] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,114] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,115] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,115] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,115] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,120] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,120] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,195] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,195] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,199] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,199] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,208] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,209] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+6: [2022-11-24 21:08:03,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:08:03,212] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:08:03,212] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,261] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,261] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,281] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,281] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,284] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,284] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+5: [2022-11-24 21:08:03,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:08:03,315] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:08:03,315] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+7: [2022-11-24 21:08:03,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:08:03,367] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:08:03,367] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now!
+0:   successfully saved checkpoint at iteration    2000 to checkpoints_1b1
+7: time (ms) | save-checkpoint: 2795.22
+7:  iteration     2010/    2891 | consumed samples:       514560 | consumed tokens:   1053818880 | elapsed time per iteration (s): 1.62 | learning rate: 5.894E-05 | global batch size:   256 | lm loss: 2.919459E+00 | grad norm: 0.270 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 158.371 | TFLOPs: 38.32 |
+7:  iteration     2020/    2891 | consumed samples:       517120 | consumed tokens:   1059061760 | elapsed time per iteration (s): 1.27 | learning rate: 5.813E-05 | global batch size:   256 | lm loss: 2.932092E+00 | grad norm: 0.275 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.012 | TFLOPs: 48.64 |
+7:  iteration     2030/    2891 | consumed samples:       519680 | consumed tokens:   1064304640 | elapsed time per iteration (s): 1.27 | learning rate: 5.733E-05 | global batch size:   256 | lm loss: 2.853943E+00 | grad norm: 0.269 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.901 | TFLOPs: 48.86 |
+7:  iteration     2040/    2891 | consumed samples:       522240 | consumed tokens:   1069547520 | elapsed time per iteration (s): 1.28 | learning rate: 5.653E-05 | global batch size:   256 | lm loss: 2.877859E+00 | grad norm: 0.253 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.940 | TFLOPs: 48.38 |
+7:  iteration     2050/    2891 | consumed samples:       524800 | consumed tokens:   1074790400 | elapsed time per iteration (s): 1.28 | learning rate: 5.574E-05 | global batch size:   256 | lm loss: 2.905728E+00 | grad norm: 0.248 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.470 | TFLOPs: 48.51 |
+7:  iteration     2060/    2891 | consumed samples:       527360 | consumed tokens:   1080033280 | elapsed time per iteration (s): 1.26 | learning rate: 5.495E-05 | global batch size:   256 | lm loss: 2.879441E+00 | grad norm: 0.245 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.919 | TFLOPs: 49.10 |
+7:  iteration     2070/    2891 | consumed samples:       529920 | consumed tokens:   1085276160 | elapsed time per iteration (s): 1.28 | learning rate: 5.418E-05 | global batch size:   256 | lm loss: 2.857717E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.535 | TFLOPs: 48.53 |
+7:  iteration     2080/    2891 | consumed samples:       532480 | consumed tokens:   1090519040 | elapsed time per iteration (s): 1.27 | learning rate: 5.340E-05 | global batch size:   256 | lm loss: 2.878311E+00 | grad norm: 0.254 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.915 | TFLOPs: 48.86 |
+7:  iteration     2090/    2891 | consumed samples:       535040 | consumed tokens:   1095761920 | elapsed time per iteration (s): 1.26 | learning rate: 5.264E-05 | global batch size:   256 | lm loss: 2.871052E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.402 | TFLOPs: 49.22 |
+7:  iteration     2100/    2891 | consumed samples:       537600 | consumed tokens:   1101004800 | elapsed time per iteration (s): 1.26 | learning rate: 5.188E-05 | global batch size:   256 | lm loss: 2.860960E+00 | grad norm: 0.264 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.576 | TFLOPs: 49.02 |
+7:  iteration     2110/    2891 | consumed samples:       540160 | consumed tokens:   1106247680 | elapsed time per iteration (s): 1.27 | learning rate: 5.113E-05 | global batch size:   256 | lm loss: 2.848944E+00 | grad norm: 0.334 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.274 | TFLOPs: 48.71 |
+7:  iteration     2120/    2891 | consumed samples:       542720 | consumed tokens:   1111490560 | elapsed time per iteration (s): 1.26 | learning rate: 5.039E-05 | global batch size:   256 | lm loss: 2.862064E+00 | grad norm: 0.257 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.001 | TFLOPs: 49.12 |
+7:  iteration     2130/    2891 | consumed samples:       545280 | consumed tokens:   1116733440 | elapsed time per iteration (s): 1.27 | learning rate: 4.965E-05 | global batch size:   256 | lm loss: 2.862977E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.476 | TFLOPs: 48.76 |
+7:  iteration     2140/    2891 | consumed samples:       547840 | consumed tokens:   1121976320 | elapsed time per iteration (s): 1.27 | learning rate: 4.892E-05 | global batch size:   256 | lm loss: 2.858904E+00 | grad norm: 0.257 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.108 | TFLOPs: 48.91 |
+7:  iteration     2150/    2891 | consumed samples:       550400 | consumed tokens:   1127219200 | elapsed time per iteration (s): 1.29 | learning rate: 4.820E-05 | global batch size:   256 | lm loss: 2.848347E+00 | grad norm: 0.266 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.874 | TFLOPs: 48.13 |
+7:  iteration     2160/    2891 | consumed samples:       552960 | consumed tokens:   1132462080 | elapsed time per iteration (s): 1.29 | learning rate: 4.749E-05 | global batch size:   256 | lm loss: 2.851582E+00 | grad norm: 0.239 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.615 | TFLOPs: 48.06 |
+7:  iteration     2170/    2891 | consumed samples:       555520 | consumed tokens:   1137704960 | elapsed time per iteration (s): 1.32 | learning rate: 4.678E-05 | global batch size:   256 | lm loss: 2.868348E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.430 | TFLOPs: 46.81 |
+7:  iteration     2180/    2891 | consumed samples:       558080 | consumed tokens:   1142947840 | elapsed time per iteration (s): 1.27 | learning rate: 4.608E-05 | global batch size:   256 | lm loss: 2.876728E+00 | grad norm: 0.265 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.566 | TFLOPs: 48.78 |
+7:  iteration     2190/    2891 | consumed samples:       560640 | consumed tokens:   1148190720 | elapsed time per iteration (s): 1.29 | learning rate: 4.539E-05 | global batch size:   256 | lm loss: 2.851987E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.987 | TFLOPs: 48.15 |
+7:  iteration     2200/    2891 | consumed samples:       563200 | consumed tokens:   1153433600 | elapsed time per iteration (s): 1.29 | learning rate: 4.471E-05 | global batch size:   256 | lm loss: 2.875356E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.981 | TFLOPs: 48.15 |
+7:  iteration     2210/    2891 | consumed samples:       565760 | consumed tokens:   1158676480 | elapsed time per iteration (s): 1.26 | learning rate: 4.403E-05 | global batch size:   256 | lm loss: 2.827237E+00 | grad norm: 0.245 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.469 | TFLOPs: 49.00 |
+7:  iteration     2220/    2891 | consumed samples:       568320 | consumed tokens:   1163919360 | elapsed time per iteration (s): 1.28 | learning rate: 4.336E-05 | global batch size:   256 | lm loss: 2.894486E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.990 | TFLOPs: 48.40 |
+7:  iteration     2230/    2891 | consumed samples:       570880 | consumed tokens:   1169162240 | elapsed time per iteration (s): 1.27 | learning rate: 4.270E-05 | global batch size:   256 | lm loss: 2.848923E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.553 | TFLOPs: 48.77 |
+7:  iteration     2240/    2891 | consumed samples:       573440 | consumed tokens:   1174405120 | elapsed time per iteration (s): 1.26 | learning rate: 4.205E-05 | global batch size:   256 | lm loss: 2.874922E+00 | grad norm: 0.256 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.808 | TFLOPs: 49.08 |
+7:  iteration     2250/    2891 | consumed samples:       576000 | consumed tokens:   1179648000 | elapsed time per iteration (s): 1.27 | learning rate: 4.141E-05 | global batch size:   256 | lm loss: 2.836610E+00 | grad norm: 0.243 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.706 | TFLOPs: 48.81 |
+7:  iteration     2260/    2891 | consumed samples:       578560 | consumed tokens:   1184890880 | elapsed time per iteration (s): 1.27 | learning rate: 4.077E-05 | global batch size:   256 | lm loss: 2.804289E+00 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.666 | TFLOPs: 48.80 |
+7:  iteration     2270/    2891 | consumed samples:       581120 | consumed tokens:   1190133760 | elapsed time per iteration (s): 1.27 | learning rate: 4.014E-05 | global batch size:   256 | lm loss: 2.808503E+00 | grad norm: 0.257 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.374 | TFLOPs: 48.73 |
+7:  iteration     2280/    2891 | consumed samples:       583680 | consumed tokens:   1195376640 | elapsed time per iteration (s): 1.29 | learning rate: 3.953E-05 | global batch size:   256 | lm loss: 2.833229E+00 | grad norm: 0.264 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.831 | TFLOPs: 48.11 |
+7:  iteration     2290/    2891 | consumed samples:       586240 | consumed tokens:   1200619520 | elapsed time per iteration (s): 1.27 | learning rate: 3.892E-05 | global batch size:   256 | lm loss: 2.864612E+00 | grad norm: 0.266 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.051 | TFLOPs: 48.65 |
+7:  iteration     2300/    2891 | consumed samples:       588800 | consumed tokens:   1205862400 | elapsed time per iteration (s): 1.29 | learning rate: 3.831E-05 | global batch size:   256 | lm loss: 2.838851E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.727 | TFLOPs: 47.85 |
+7:  iteration     2310/    2891 | consumed samples:       591360 | consumed tokens:   1211105280 | elapsed time per iteration (s): 1.28 | learning rate: 3.772E-05 | global batch size:   256 | lm loss: 2.862267E+00 | grad norm: 0.259 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.488 | TFLOPs: 48.52 |
+7:  iteration     2320/    2891 | consumed samples:       593920 | consumed tokens:   1216348160 | elapsed time per iteration (s): 1.27 | learning rate: 3.714E-05 | global batch size:   256 | lm loss: 2.854469E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.255 | TFLOPs: 48.70 |
+7:  iteration     2330/    2891 | consumed samples:       596480 | consumed tokens:   1221591040 | elapsed time per iteration (s): 1.28 | learning rate: 3.656E-05 | global batch size:   256 | lm loss: 2.785706E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.445 | TFLOPs: 48.26 |
+7:  iteration     2340/    2891 | consumed samples:       599040 | consumed tokens:   1226833920 | elapsed time per iteration (s): 1.27 | learning rate: 3.600E-05 | global batch size:   256 | lm loss: 2.798930E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.959 | TFLOPs: 48.63 |
+7:  iteration     2350/    2891 | consumed samples:       601600 | consumed tokens:   1232076800 | elapsed time per iteration (s): 1.27 | learning rate: 3.544E-05 | global batch size:   256 | lm loss: 2.827826E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.774 | TFLOPs: 48.83 |
+7:  iteration     2360/    2891 | consumed samples:       604160 | consumed tokens:   1237319680 | elapsed time per iteration (s): 1.28 | learning rate: 3.489E-05 | global batch size:   256 | lm loss: 2.845606E+00 | grad norm: 0.253 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.486 | TFLOPs: 48.27 |
+7:  iteration     2370/    2891 | consumed samples:       606720 | consumed tokens:   1242562560 | elapsed time per iteration (s): 1.29 | learning rate: 3.435E-05 | global batch size:   256 | lm loss: 2.854224E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.010 | TFLOPs: 48.16 |
+7:  iteration     2380/    2891 | consumed samples:       609280 | consumed tokens:   1247805440 | elapsed time per iteration (s): 1.29 | learning rate: 3.382E-05 | global batch size:   256 | lm loss: 2.833812E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.147 | TFLOPs: 48.19 |
+7:  iteration     2390/    2891 | consumed samples:       611840 | consumed tokens:   1253048320 | elapsed time per iteration (s): 1.28 | learning rate: 3.330E-05 | global batch size:   256 | lm loss: 2.822174E+00 | grad norm: 0.262 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.528 | TFLOPs: 48.28 |
+7:  iteration     2400/    2891 | consumed samples:       614400 | consumed tokens:   1258291200 | elapsed time per iteration (s): 1.27 | learning rate: 3.279E-05 | global batch size:   256 | lm loss: 2.868549E+00 | grad norm: 0.248 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.976 | TFLOPs: 48.63 |
+7:  iteration     2410/    2891 | consumed samples:       616960 | consumed tokens:   1263534080 | elapsed time per iteration (s): 1.27 | learning rate: 3.228E-05 | global batch size:   256 | lm loss: 2.831634E+00 | grad norm: 0.267 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.641 | TFLOPs: 48.80 |
+7:  iteration     2420/    2891 | consumed samples:       619520 | consumed tokens:   1268776960 | elapsed time per iteration (s): 1.28 | learning rate: 3.179E-05 | global batch size:   256 | lm loss: 2.823357E+00 | grad norm: 0.239 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.067 | TFLOPs: 48.41 |
+7:  iteration     2430/    2891 | consumed samples:       622080 | consumed tokens:   1274019840 | elapsed time per iteration (s): 1.26 | learning rate: 3.131E-05 | global batch size:   256 | lm loss: 2.828638E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.396 | TFLOPs: 48.98 |
+7:  iteration     2440/    2891 | consumed samples:       624640 | consumed tokens:   1279262720 | elapsed time per iteration (s): 1.27 | learning rate: 3.083E-05 | global batch size:   256 | lm loss: 2.800687E+00 | grad norm: 0.245 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.414 | TFLOPs: 48.74 |
+7:  iteration     2450/    2891 | consumed samples:       627200 | consumed tokens:   1284505600 | elapsed time per iteration (s): 1.26 | learning rate: 3.037E-05 | global batch size:   256 | lm loss: 2.824464E+00 | grad norm: 0.244 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.599 | TFLOPs: 49.27 |
+7:  iteration     2460/    2891 | consumed samples:       629760 | consumed tokens:   1289748480 | elapsed time per iteration (s): 1.29 | learning rate: 2.991E-05 | global batch size:   256 | lm loss: 2.851115E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.887 | TFLOPs: 48.13 |
+7:  iteration     2470/    2891 | consumed samples:       632320 | consumed tokens:   1294991360 | elapsed time per iteration (s): 1.26 | learning rate: 2.947E-05 | global batch size:   256 | lm loss: 2.820526E+00 | grad norm: 0.254 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.981 | TFLOPs: 49.12 |
+7:  iteration     2480/    2891 | consumed samples:       634880 | consumed tokens:   1300234240 | elapsed time per iteration (s): 1.27 | learning rate: 2.903E-05 | global batch size:   256 | lm loss: 2.799962E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.887 | TFLOPs: 48.85 |
+7:  iteration     2490/    2891 | consumed samples:       637440 | consumed tokens:   1305477120 | elapsed time per iteration (s): 1.27 | learning rate: 2.860E-05 | global batch size:   256 | lm loss: 2.799555E+00 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.814 | TFLOPs: 48.59 |
+7:  iteration     2500/    2891 | consumed samples:       640000 | consumed tokens:   1310720000 | elapsed time per iteration (s): 1.26 | learning rate: 2.819E-05 | global batch size:   256 | lm loss: 2.822092E+00 | grad norm: 0.256 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.382 | TFLOPs: 49.22 |
+7:  iteration     2510/    2891 | consumed samples:       642560 | consumed tokens:   1315962880 | elapsed time per iteration (s): 1.27 | learning rate: 2.778E-05 | global batch size:   256 | lm loss: 2.835498E+00 | grad norm: 0.247 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.136 | TFLOPs: 48.67 |
+7:  iteration     2520/    2891 | consumed samples:       645120 | consumed tokens:   1321205760 | elapsed time per iteration (s): 1.27 | learning rate: 2.738E-05 | global batch size:   256 | lm loss: 2.794395E+00 | grad norm: 0.235 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.216 | TFLOPs: 48.69 |
+7:  iteration     2530/    2891 | consumed samples:       647680 | consumed tokens:   1326448640 | elapsed time per iteration (s): 1.27 | learning rate: 2.700E-05 | global batch size:   256 | lm loss: 2.808821E+00 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.007 | TFLOPs: 48.88 |
+7:  iteration     2540/    2891 | consumed samples:       650240 | consumed tokens:   1331691520 | elapsed time per iteration (s): 1.27 | learning rate: 2.662E-05 | global batch size:   256 | lm loss: 2.793146E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.216 | TFLOPs: 48.69 |
+7:  iteration     2550/    2891 | consumed samples:       652800 | consumed tokens:   1336934400 | elapsed time per iteration (s): 1.28 | learning rate: 2.625E-05 | global batch size:   256 | lm loss: 2.770286E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.289 | TFLOPs: 48.47 |
+7:  iteration     2560/    2891 | consumed samples:       655360 | consumed tokens:   1342177280 | elapsed time per iteration (s): 1.28 | learning rate: 2.590E-05 | global batch size:   256 | lm loss: 2.784430E+00 | grad norm: 0.259 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.743 | TFLOPs: 48.34 |
+7:  iteration     2570/    2891 | consumed samples:       657920 | consumed tokens:   1347420160 | elapsed time per iteration (s): 1.26 | learning rate: 2.555E-05 | global batch size:   256 | lm loss: 2.799257E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 203.123 | TFLOPs: 49.15 |
+7:  iteration     2580/    2891 | consumed samples:       660480 | consumed tokens:   1352663040 | elapsed time per iteration (s): 1.27 | learning rate: 2.521E-05 | global batch size:   256 | lm loss: 2.804961E+00 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.170 | TFLOPs: 48.68 |
+7:  iteration     2590/    2891 | consumed samples:       663040 | consumed tokens:   1357905920 | elapsed time per iteration (s): 1.28 | learning rate: 2.489E-05 | global batch size:   256 | lm loss: 2.828386E+00 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.053 | TFLOPs: 48.41 |
+7:  iteration     2600/    2891 | consumed samples:       665600 | consumed tokens:   1363148800 | elapsed time per iteration (s): 1.27 | learning rate: 2.457E-05 | global batch size:   256 | lm loss: 2.777740E+00 | grad norm: 0.253 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.953 | TFLOPs: 48.87 |
+7:  iteration     2610/    2891 | consumed samples:       668160 | consumed tokens:   1368391680 | elapsed time per iteration (s): 1.27 | learning rate: 2.427E-05 | global batch size:   256 | lm loss: 2.815404E+00 | grad norm: 0.243 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.950 | TFLOPs: 48.63 |
+7:  iteration     2620/    2891 | consumed samples:       670720 | consumed tokens:   1373634560 | elapsed time per iteration (s): 1.28 | learning rate: 2.397E-05 | global batch size:   256 | lm loss: 2.825298E+00 | grad norm: 0.247 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.339 | TFLOPs: 48.48 |
+7:  iteration     2630/    2891 | consumed samples:       673280 | consumed tokens:   1378877440 | elapsed time per iteration (s): 1.27 | learning rate: 2.369E-05 | global batch size:   256 | lm loss: 2.799451E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.902 | TFLOPs: 48.86 |
+7:  iteration     2640/    2891 | consumed samples:       675840 | consumed tokens:   1384120320 | elapsed time per iteration (s): 1.28 | learning rate: 2.341E-05 | global batch size:   256 | lm loss: 2.847309E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.515 | TFLOPs: 48.52 |
+7:  iteration     2650/    2891 | consumed samples:       678400 | consumed tokens:   1389363200 | elapsed time per iteration (s): 1.28 | learning rate: 2.315E-05 | global batch size:   256 | lm loss: 2.814650E+00 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.473 | TFLOPs: 48.27 |
+7:  iteration     2660/    2891 | consumed samples:       680960 | consumed tokens:   1394606080 | elapsed time per iteration (s): 1.28 | learning rate: 2.289E-05 | global batch size:   256 | lm loss: 2.785754E+00 | grad norm: 0.248 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.125 | TFLOPs: 48.43 |
+7:  iteration     2670/    2891 | consumed samples:       683520 | consumed tokens:   1399848960 | elapsed time per iteration (s): 1.30 | learning rate: 2.265E-05 | global batch size:   256 | lm loss: 2.793461E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.358 | TFLOPs: 47.76 |
+7:  iteration     2680/    2891 | consumed samples:       686080 | consumed tokens:   1405091840 | elapsed time per iteration (s): 1.28 | learning rate: 2.242E-05 | global batch size:   256 | lm loss: 2.760545E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.544 | TFLOPs: 48.53 |
+7:  iteration     2690/    2891 | consumed samples:       688640 | consumed tokens:   1410334720 | elapsed time per iteration (s): 1.27 | learning rate: 2.220E-05 | global batch size:   256 | lm loss: 2.787152E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.989 | TFLOPs: 48.64 |
+7:  iteration     2700/    2891 | consumed samples:       691200 | consumed tokens:   1415577600 | elapsed time per iteration (s): 1.28 | learning rate: 2.198E-05 | global batch size:   256 | lm loss: 2.798705E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.327 | TFLOPs: 48.24 |
+7:  iteration     2710/    2891 | consumed samples:       693760 | consumed tokens:   1420820480 | elapsed time per iteration (s): 1.27 | learning rate: 2.178E-05 | global batch size:   256 | lm loss: 2.760976E+00 | grad norm: 0.250 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.992 | TFLOPs: 48.64 |
+7:  iteration     2720/    2891 | consumed samples:       696320 | consumed tokens:   1426063360 | elapsed time per iteration (s): 1.27 | learning rate: 2.159E-05 | global batch size:   256 | lm loss: 2.821980E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.479 | TFLOPs: 48.76 |
+7:  iteration     2730/    2891 | consumed samples:       698880 | consumed tokens:   1431306240 | elapsed time per iteration (s): 1.26 | learning rate: 2.141E-05 | global batch size:   256 | lm loss: 2.780820E+00 | grad norm: 0.258 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.844 | TFLOPs: 49.09 |
+7:  iteration     2740/    2891 | consumed samples:       701440 | consumed tokens:   1436549120 | elapsed time per iteration (s): 1.29 | learning rate: 2.124E-05 | global batch size:   256 | lm loss: 2.757240E+00 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.089 | TFLOPs: 48.18 |
+7:  iteration     2750/    2891 | consumed samples:       704000 | consumed tokens:   1441792000 | elapsed time per iteration (s): 1.29 | learning rate: 2.109E-05 | global batch size:   256 | lm loss: 2.767885E+00 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 197.871 | TFLOPs: 47.88 |
+7:  iteration     2760/    2891 | consumed samples:       706560 | consumed tokens:   1447034880 | elapsed time per iteration (s): 1.26 | learning rate: 2.094E-05 | global batch size:   256 | lm loss: 2.782544E+00 | grad norm: 0.277 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.573 | TFLOPs: 49.02 |
+7:  iteration     2770/    2891 | consumed samples:       709120 | consumed tokens:   1452277760 | elapsed time per iteration (s): 1.27 | learning rate: 2.080E-05 | global batch size:   256 | lm loss: 2.807351E+00 | grad norm: 0.258 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.197 | TFLOPs: 48.69 |
+7:  iteration     2780/    2891 | consumed samples:       711680 | consumed tokens:   1457520640 | elapsed time per iteration (s): 1.32 | learning rate: 2.068E-05 | global batch size:   256 | lm loss: 2.795422E+00 | grad norm: 0.259 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 193.852 | TFLOPs: 46.91 |
+7:  iteration     2790/    2891 | consumed samples:       714240 | consumed tokens:   1462763520 | elapsed time per iteration (s): 1.27 | learning rate: 2.056E-05 | global batch size:   256 | lm loss: 2.809049E+00 | grad norm: 0.261 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.671 | TFLOPs: 48.80 |
+7:  iteration     2800/    2891 | consumed samples:       716800 | consumed tokens:   1468006400 | elapsed time per iteration (s): 1.28 | learning rate: 2.046E-05 | global batch size:   256 | lm loss: 2.831892E+00 | grad norm: 0.249 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.506 | TFLOPs: 48.52 |
+7:  iteration     2810/    2891 | consumed samples:       719360 | consumed tokens:   1473249280 | elapsed time per iteration (s): 1.29 | learning rate: 2.036E-05 | global batch size:   256 | lm loss: 2.751667E+00 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 198.767 | TFLOPs: 48.10 |
+7:  iteration     2820/    2891 | consumed samples:       721920 | consumed tokens:   1478492160 | elapsed time per iteration (s): 1.27 | learning rate: 2.028E-05 | global batch size:   256 | lm loss: 2.827281E+00 | grad norm: 0.256 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.891 | TFLOPs: 48.86 |
+7:  iteration     2830/    2891 | consumed samples:       724480 | consumed tokens:   1483735040 | elapsed time per iteration (s): 1.30 | learning rate: 2.021E-05 | global batch size:   256 | lm loss: 2.819973E+00 | grad norm: 0.260 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 196.456 | TFLOPs: 47.54 |
+7:  iteration     2840/    2891 | consumed samples:       727040 | consumed tokens:   1488977920 | elapsed time per iteration (s): 1.28 | learning rate: 2.014E-05 | global batch size:   256 | lm loss: 2.776796E+00 | grad norm: 0.281 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 199.591 | TFLOPs: 48.30 |
+7:  iteration     2850/    2891 | consumed samples:       729600 | consumed tokens:   1494220800 | elapsed time per iteration (s): 1.27 | learning rate: 2.009E-05 | global batch size:   256 | lm loss: 2.797709E+00 | grad norm: 0.252 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 201.425 | TFLOPs: 48.74 |
+7:  iteration     2860/    2891 | consumed samples:       732160 | consumed tokens:   1499463680 | elapsed time per iteration (s): 1.27 | learning rate: 2.005E-05 | global batch size:   256 | lm loss: 2.794808E+00 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.000 | TFLOPs: 48.88 |
+7:  iteration     2870/    2891 | consumed samples:       734720 | consumed tokens:   1504706560 | elapsed time per iteration (s): 1.28 | learning rate: 2.003E-05 | global batch size:   256 | lm loss: 2.803428E+00 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.153 | TFLOPs: 48.44 |
+7:  iteration     2880/    2891 | consumed samples:       737280 | consumed tokens:   1509949440 | elapsed time per iteration (s): 1.26 | learning rate: 2.001E-05 | global batch size:   256 | lm loss: 2.790293E+00 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 202.895 | TFLOPs: 49.10 |
+7:  iteration     2890/    2891 | consumed samples:       739840 | consumed tokens:   1515192320 | elapsed time per iteration (s): 1.28 | learning rate: 2.000E-05 | global batch size:   256 | lm loss: 2.775094E+00 | grad norm: 0.258 | num zeros: 0.0 | number of skipped iterations:   0 | number of nan iterations:   0 | samples per second: 200.463 | TFLOPs: 48.51 |
+0: [after training is done] datetime: 2022-11-24 21:27:00 
+0: saving checkpoint at iteration    2891 to checkpoints_1b1
+7: ------------------------------------------------------------------------------------------------------------
+7: valid loss at the end of training for val data | lm loss value: 2.731300E+00 | lm loss PPL: 1.535284E+01 | 
+7: ------------------------------------------------------------------------------------------------------------
+0: [2022-11-24 21:27:00,642] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2891 is begin to save!
+0: [2022-11-24 21:27:00,645] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_01-model_00-model_states.pt...
+0: [2022-11-24 21:27:00,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_01-model_00-model_states.pt.
+0: [2022-11-24 21:27:00,838] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_03-model_00-model_states.pt...
+0: [2022-11-24 21:27:00,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_03-model_00-model_states.pt.
+0: [2022-11-24 21:27:00,915] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_04-model_00-model_states.pt...
+0: [2022-11-24 21:27:00,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_04-model_00-model_states.pt.
+0: [2022-11-24 21:27:00,988] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_05-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,061] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_05-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,062] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_06-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,135] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_06-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,135] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_07-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,208] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_07-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,208] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_08-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,282] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_08-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,282] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_09-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_09-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,355] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_10-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_10-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,424] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_11-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,501] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_11-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,501] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_12-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,575] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_12-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,576] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_13-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_13-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,650] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_14-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_14-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,723] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_15-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,795] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_15-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,796] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_16-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_16-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,866] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_17-model_00-model_states.pt...
+0: [2022-11-24 21:27:01,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_17-model_00-model_states.pt.
+0: [2022-11-24 21:27:01,940] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_18-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_18-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,016] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_19-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,090] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_19-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,090] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_20-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,161] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_20-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,162] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_21-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_21-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,236] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_22-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_22-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,310] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_23-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_23-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,384] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_24-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_24-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_25-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_25-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,528] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_26-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,600] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_26-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,601] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_27-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_27-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,674] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_28-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_28-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,750] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/layer_30-model_00-model_states.pt...
+0: [2022-11-24 21:27:02,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/layer_30-model_00-model_states.pt.
+0: [2022-11-24 21:27:02,752] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: checkpoints_1b1/global_step2891/mp_rank_00_model_states.pt
+0: [2022-11-24 21:27:02,752] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/mp_rank_00_model_states.pt...
+0: [2022-11-24 21:27:02,755] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/mp_rank_00_model_states.pt.
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt...
+0: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt...
+1: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt...
+7: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt...
+4: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt...
+3: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt...
+2: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt...
+5: [2022-11-24 21:27:02,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving checkpoints_1b1/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt...
+6: [2022-11-24 21:27:03,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,016] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,016] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,022] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,022] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,023] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,023] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,029] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,029] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,030] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,030] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,030] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,030] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,032] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,032] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,032] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,038] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,038] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,042] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,042] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,043] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,043] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,043] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,043] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,028] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,022] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,028] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,022] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,038] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,038] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,039] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,039] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,039] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,039] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,046] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,046] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,051] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,051] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,054] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,054] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,055] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,055] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,055] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,055] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt.
+4: [2022-11-24 21:27:03,056] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,056] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,056] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+4: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,056] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,057] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,057] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,057] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,063] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,063] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,063] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,063] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,068] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,068] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,068] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,068] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,068] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,069] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,069] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,069] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,070] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,070] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,073] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,073] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,077] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,077] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+3: [2022-11-24 21:27:03,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt.
+3: [2022-11-24 21:27:03,078] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
+3: [2022-11-24 21:27:03,078] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.
+0: [2022-11-24 21:27:03,087] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,087] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,054] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,055] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,055] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,054] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,055] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,056] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+2: [2022-11-24 21:27:03,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.
+2: [2022-11-24 21:27:03,065] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
+2: [2022-11-24 21:27:03,065] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,102] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,102] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,102] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,102] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+1: [2022-11-24 21:27:03,108] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.
+1: [2022-11-24 21:27:03,108] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
+1: [2022-11-24 21:27:03,109] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,110] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,110] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,110] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,149] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,149] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,149] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0: [2022-11-24 21:27:03,153] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+0: [2022-11-24 21:27:03,153] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,170] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,170] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,176] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,176] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,187] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,188] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,188] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,191] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,191] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,188] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,188] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,188] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,224] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,224] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,219] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,219] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,232] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,232] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,232] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,236] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,237] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,291] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,291] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,291] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,306] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,306] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,323] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,323] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,324] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+6: [2022-11-24 21:27:03,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt.
+6: [2022-11-24 21:27:03,359] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt
+6: [2022-11-24 21:27:03,359] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+7: [2022-11-24 21:27:03,370] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt.
+7: [2022-11-24 21:27:03,370] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt
+7: [2022-11-24 21:27:03,370] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+5: [2022-11-24 21:27:03,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt.
+5: [2022-11-24 21:27:03,374] [INFO] [engine.py:3213:_save_zero_checkpoint] bf16_zero checkpoint saved checkpoints_1b1/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
+5: [2022-11-24 21:27:03,374] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2891 is ready now!
+0:   successfully saved checkpoint at iteration    2891 to checkpoints_1b1
+7: ------------------------------------------------------------------------------------------------------------
+7: test loss at the end of training for test data | lm loss value: 2.712490E+00 | lm loss PPL: 1.506675E+01 | 
+7: ------------------------------------------------------------------------------------------------------------
+END 2068467: Thu Nov 24 21:27:07 EET 2022
diff --git a/1b11b51b5/logs/2820936.err b/1b11b51b5/logs/2820936.err
new file mode 100644
index 0000000000000000000000000000000000000000..30cb4a88ac88335638c2d9c5f609eb6b690fb214
--- /dev/null
+++ b/1b11b51b5/logs/2820936.err
@@ -0,0 +1,1119 @@
+7: 2023-02-09 23:01:23.406563: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406572: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406584: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406613: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406624: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406636: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406645: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+7: 2023-02-09 23:01:23.406719: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407378: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407390: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407403: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407408: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407559: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: 2023-02-09 23:01:23.407610: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407633: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+6: 2023-02-09 23:01:23.407629: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407710: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407729: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:23.407716: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414873: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414894: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414912: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414941: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414958: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414932: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.414964: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+1: 2023-02-09 23:01:23.415018: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431578: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431587: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431600: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431607: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+0: 2023-02-09 23:01:23.431623: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432407: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432442: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: 2023-02-09 23:01:23.432469: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432501: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432470: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432519: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432495: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432556: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432629: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432439: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432452: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+5: 2023-02-09 23:01:23.432660: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432460: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432434: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+3: 2023-02-09 23:01:23.432477: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433311: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433350: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433319: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433357: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433387: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433404: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+4: 2023-02-09 23:01:23.433406: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA
+4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2: 2023-02-09 23:01:25.019084: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019086: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+2: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:25.019305: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019306: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019310: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019305: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019312: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019313: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019314: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+2: 2023-02-09 23:01:25.019317: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020846: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+6: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:25.020849: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020853: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020853: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020855: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+6: 2023-02-09 23:01:25.020863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.037713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.037736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+5: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:25.038123: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038126: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038131: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038133: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038137: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038139: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038145: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+5: 2023-02-09 23:01:25.038146: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+7: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:25.114747: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114753: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114754: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114758: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114762: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114762: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+7: 2023-02-09 23:01:25.114765: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+3: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:25.151754: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151759: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151761: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151763: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151764: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151765: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+3: 2023-02-09 23:01:25.151769: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+4: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:25.153507: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153510: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153511: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153510: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153511: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153516: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153516: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:25.153523: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169172: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169172: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169172: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+0: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:25.169566: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169567: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169573: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169574: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169575: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169576: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169577: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+0: 2023-02-09 23:01:25.169579: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.249718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249723: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249729: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.249755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_46200
+1: 0125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:25.250244: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250248: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250250: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250254: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250256: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250258: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250264: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+1: 2023-02-09 23:01:25.250262: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
+4: 2023-02-09 23:01:42.122638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.122772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+4: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125214: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125215: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125212: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+4: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+4: 2023-02-09 23:01:42.125222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125230: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125228: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125235: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+4: 2023-02-09 23:01:42.125237: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.130892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.130983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.131121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+5: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132879: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132885: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132894: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132894: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+5: 2023-02-09 23:01:42.132927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+5: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+5: 2023-02-09 23:01:42.132945: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+6: 2023-02-09 23:01:42.139586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139712: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.139744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: 2023-02-09 23:01:42.139631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: 2023-02-09 23:01:42.139804: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.139864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: 2023-02-09 23:01:42.139780: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139853: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.139857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.139682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.139983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+6: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.139881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: 2023-02-09 23:01:42.139862: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+0: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.139854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: 2023-02-09 23:01:42.139894: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: 2023-02-09 23:01:42.139900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+2: 2023-02-09 23:01:42.139941: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: 2023-02-09 23:01:42.139915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+1: 2023-02-09 23:01:42.139927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+7: 2023-02-09 23:01:42.139952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.139983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/project_462000125
+3: /samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: 2023-02-09 23:01:42.142039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: 2023-02-09 23:01:42.142045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142049: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+6: 2023-02-09 23:01:42.142050: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: 2023-02-09 23:01:42.142059: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+6: 2023-02-09 23:01:42.142062: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+6: 2023-02-09 23:01:42.142063: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: 2023-02-09 23:01:42.142063: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+6: 2023-02-09 23:01:42.142066: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: 2023-02-09 23:01:42.142194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+6: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+6: 2023-02-09 23:01:42.142207: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142172: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142174: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142180: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+7: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+7: 2023-02-09 23:01:42.142227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+7: 2023-02-09 23:01:42.142228: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.142408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142411: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.142408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142411: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142409: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: 2023-02-09 23:01:42.142630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142412: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: 2023-02-09 23:01:42.142631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.142415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: 2023-02-09 23:01:42.142512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: 2023-02-09 23:01:42.142632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.142423: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: 2023-02-09 23:01:42.142428: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142521: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: 2023-02-09 23:01:42.142520: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+3: 2023-02-09 23:01:42.142428: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: 2023-02-09 23:01:42.142430: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: 2023-02-09 23:01:42.142431: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+3: 2023-02-09 23:01:42.142431: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: 2023-02-09 23:01:42.142434: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+3: 2023-02-09 23:01:42.142435: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:42.142635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:42.142635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:42.142638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:42.142638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+1: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+1: 2023-02-09 23:01:42.142646: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142648: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142648: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142652: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142654: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+1: 2023-02-09 23:01:42.142656: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141856: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+2: 2023-02-09 23:01:42.142520: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: 2023-02-09 23:01:42.142527: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: 2023-02-09 23:01:42.142528: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+2: 2023-02-09 23:01:42.142531: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: 2023-02-09 23:01:42.142529: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+2: 2023-02-09 23:01:42.142531: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141872: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141874: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141880: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141881: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141881: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141883: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: 2023-02-09 23:01:42.141889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps/aws-ofi-rccl:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/rccl/rccl-develop-release/rccl/lib:/pfs/lustrep4/projappl/project_462000075/samantao-public/rocm/glibc/selected:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hip/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/hsa/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/llvm:/pfs/lustrep2/projappl/pro
+0: ject_462000125/samantao-public/apps/suse-repo-deps/lib64:/pfs/lustrep2/projappl/project_462000125/samantao-public/apps/suse-repo-deps/usr/lib64:/opt/cray/pe/python/3.9.12.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
+0: 2023-02-09 23:01:42.141904: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module scaled_upper_triang_masked_softmax_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module scaled_upper_triang_masked_softmax_cuda...
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module scaled_masked_softmax_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module scaled_masked_softmax_cuda...
+0: Successfully preprocessed all matching files.
+0: Detected CUDA files, patching ldflags
+0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja...
+0: Building extension module fused_mix_prec_layer_norm_cuda...
+0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Loading extension module fused_mix_prec_layer_norm_cuda...
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+0: Successfully preprocessed all matching files.
+7: Successfully preprocessed all matching files.
+7: Successfully preprocessed all matching files.
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+7:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+3:   warnings.warn(
+5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+5:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+4:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+1:   warnings.warn(
+6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+6:   warnings.warn(
+2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+2:   warnings.warn(
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead
+0:   warnings.warn(
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja...
+7: Building extension module utils...
+7: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: 
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+1: 
+1: 
+1: 
+1: 
+1: 
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: 
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: 
+2: 
+2: 
+2: 
+2: 
+2: 
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: 
+3: 
+3: 
+3: 
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: 
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: 
+4: 
+4: 
+4: 
+7: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+0: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+5: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+7: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+6: Loading extension module utils...
+7: Loading extension module utils...
+6: Loading extension module utils...
+6: Loading extension module utils...
+1: Loading extension module utils...
+1: Loading extension module utils...
+1: Loading extension module utils...
+2: Loading extension module utils...
+1: Loading extension module utils...
+2: Loading extension module utils...
+1: Loading extension module utils...
+3: Loading extension module utils...
+2: Loading extension module utils...
+1: Loading extension module utils...
+3: Loading extension module utils...
+1: Loading extension module utils...
+3: Loading extension module utils...
+2: Loading extension module utils...
+1: Loading extension module utils...
+2: Loading extension module utils...
+3: Loading extension module utils...
+4: Loading extension module utils...
+2: Loading extension module utils...
+3: Loading extension module utils...
+2: Loading extension module utils...
+3: Loading extension module utils...
+2: Loading extension module utils...
+3: Loading extension module utils...
+4: Loading extension module utils...
+3: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+4: Loading extension module utils...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: 
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+7: No modifications detected for re-loaded extension module utils, skipping build step...
+7: Loading extension module utils...
+0: Loading extension module utils...
+6: Loading extension module utils...
+5: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: 
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: No modifications detected for re-loaded extension module utils, skipping build step...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: 
+6: Loading extension module utils...
+6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...
+6: 
+6: 
+6: 
+6: Loading extension module utils...
+6: Loading extension module utils...Loading extension module utils...Loading extension module utils...
+6: 
+6: 
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: 
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+6: No modifications detected for re-loaded extension module utils, skipping build step...
+6: Loading extension module utils...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: 
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: 
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: 
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: 
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: 
+1: 
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...
+5: 
+5: Loading extension module utils...Loading extension module utils...
+5: 
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+5: No modifications detected for re-loaded extension module utils, skipping build step...
+5: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+1: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+1: No modifications detected for re-loaded extension module utils, skipping build step...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+1: Loading extension module utils...
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...
+4: 
+4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...
+4: 
+4: Loading extension module utils...
+4: No modifications detected for re-loaded extension module utils, skipping build step...
+4: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+2: No modifications detected for re-loaded extension module utils, skipping build step...
+2: Loading extension module utils...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: 
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...
+3: 
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+3: No modifications detected for re-loaded extension module utils, skipping build step...
+3: Loading extension module utils...
+0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...
+0: No modifications detected for re-loaded extension module utils, skipping build step...
+0: Loading extension module utils...
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings
+0:   warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings")
diff --git a/1b11b51b5/logs/2820936.out b/1b11b51b5/logs/2820936.out
new file mode 100644
index 0000000000000000000000000000000000000000..d65610b385245bcc827d48060dbdc1605338de79
--- /dev/null
+++ b/1b11b51b5/logs/2820936.out
@@ -0,0 +1,8491 @@
+Model parameters: d_model 1792 ffw_size 7168 kv_size 128 n_heads 14 n_layers 26
+Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --num-layers 26 --hidden-size 1792 --num-attention-heads 14 --kv-channels 128 --ffn-hidden-size 7168 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 4 --global-batch-size 256 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --loss-scale 12 --clip-grad 1.0 --kill-switch-path kill-switch-1b11b51b5val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --no-load-optim --reset-progress --override-lr-scheduler --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-only true --eval-iters 100 --tensorboard-dir tensorboard_1b11b51b5val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save checkpoints_1b11b51b5 --load checkpoints_1b11b51b5 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/2820936.json --zero-stage 0
+START 2820936: Thu 09 Feb 2023 11:00:36 PM EET
+0: 
+0: 
+0: ======================= ROCm System Management Interface =======================
+0: ================================= Concise Info =================================
+0: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+0: 0    40.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 1    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 2    42.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 3    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 4    43.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 5    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: 6    40.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+0: 7    40.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+0: ================================================================================
+0: ============================= End of ROCm SMI Log ==============================
+2: 
+2: 
+2: ======================= ROCm System Management Interface =======================
+2: ================================= Concise Info =================================
+2: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+2: 0    43.0c  98.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 1    42.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 2    43.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 3    50.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 4    42.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 5    47.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: 6    34.0c  88.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+2: 7    38.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+2: ================================================================================
+2: ============================= End of ROCm SMI Log ==============================
+5: 
+5: 
+5: ======================= ROCm System Management Interface =======================
+5: ================================= Concise Info =================================
+5: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+5: 0    43.0c  92.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 1    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 2    36.0c  83.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 3    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 4    43.0c  79.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 5    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: 6    39.0c  91.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+5: 7    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+5: ================================================================================
+5: ============================= End of ROCm SMI Log ==============================
+1: 
+1: 
+1: ======================= ROCm System Management Interface =======================
+1: ================================= Concise Info =================================
+1: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+1: 0    42.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 1    47.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 2    41.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 3    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 4    43.0c  91.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 5    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: 6    39.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+1: 7    40.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+1: ================================================================================
+1: ============================= End of ROCm SMI Log ==============================
+4: 
+4: 
+4: ======================= ROCm System Management Interface =======================
+4: ================================= Concise Info =================================
+4: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+4: 0    49.0c  90.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 2    40.0c  81.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 3    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 4    37.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 5    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: 6    36.0c  81.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+4: 7    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+4: ================================================================================
+4: ============================= End of ROCm SMI Log ==============================
+6: 
+6: 
+6: ======================= ROCm System Management Interface =======================
+6: ================================= Concise Info =================================
+6: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+6: 0    47.0c  85.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 2    37.0c  84.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 3    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 4    38.0c  88.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 5    46.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: 6    45.0c  83.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+6: 7    41.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+6: ================================================================================
+6: ============================= End of ROCm SMI Log ==============================
+7: 
+7: 
+7: ======================= ROCm System Management Interface =======================
+7: ================================= Concise Info =================================
+7: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+7: 0    46.0c  87.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 2    41.0c  83.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 3    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 4    45.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 5    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: 6    41.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+7: 7    43.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+7: ================================================================================
+7: ============================= End of ROCm SMI Log ==============================
+3: 
+3: 
+3: ======================= ROCm System Management Interface =======================
+3: ================================= Concise Info =================================
+3: GPU  Temp   AvgPwr  SCLK    MCLK     Fan  Perf  PwrCap  VRAM%  GPU%  
+3: 0    40.0c  91.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 1    45.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 2    42.0c  94.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 3    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 4    45.0c  81.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 5    48.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: 6    40.0c  86.0W   800Mhz  1600Mhz  0%   auto  560.0W    0%   0%    
+3: 7    44.0c  N/A     800Mhz  1600Mhz  0%   auto  0.0W      0%   0%    
+3: ================================================================================
+3: ============================= End of ROCm SMI Log ==============================
+0: Launching on nid005004 (0/8), master nid005004 port 9999, GPUs 8, CUDA: True
+1: Launching on nid005005 (1/8), master nid005004 port 9999, GPUs 8, CUDA: True
+7: Launching on nid005011 (7/8), master nid005004 port 9999, GPUs 8, CUDA: True
+4: Launching on nid005008 (4/8), master nid005004 port 9999, GPUs 8, CUDA: True
+5: Launching on nid005009 (5/8), master nid005004 port 9999, GPUs 8, CUDA: True
+2: Launching on nid005006 (2/8), master nid005004 port 9999, GPUs 8, CUDA: True
+6: Launching on nid005010 (6/8), master nid005004 port 9999, GPUs 8, CUDA: True
+3: Launching on nid005007 (3/8), master nid005004 port 9999, GPUs 8, CUDA: True
+0: using world size: 64, data-parallel-size: 64, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 
+0: accumulate and all-reduce gradients in fp32 for bfloat16 data type.
+0: using torch.bfloat16 for parameters ...
+0: ------------------------ arguments ------------------------
+0:   abort_on_unmet_fused_kernel_constraints ......... False
+0:   accumulate_allreduce_grads_in_fp32 .............. True
+0:   adam_beta1 ...................................... 0.9
+0:   adam_beta2 ...................................... 0.999
+0:   adam_eps ........................................ 1e-08
+0:   adlr_autoresume ................................. False
+0:   adlr_autoresume_interval ........................ 1000
+0:   apply_query_key_layer_scaling ................... True
+0:   apply_residual_connection_post_layernorm ........ False
+0:   attention_dropout ............................... 0.1
+0:   attention_softmax_in_fp32 ....................... False
+0:   bert_binary_head ................................ True
+0:   bert_load ....................................... None
+0:   bf16 ............................................ True
+0:   bias_dropout_fusion ............................. True
+0:   bias_gelu_fusion ................................ True
+0:   biencoder_projection_dim ........................ 0
+0:   biencoder_shared_query_context_model ............ False
+0:   block_data_path ................................. None
+0:   checkpoint_activations .......................... False
+0:   checkpoint_in_cpu ............................... False
+0:   checkpoint_num_layers ........................... 1
+0:   clip_grad ....................................... 1.0
+0:   codecarbon_dir .................................. None
+0:   consumed_train_samples .......................... 0
+0:   consumed_train_tokens ........................... 0
+0:   consumed_valid_samples .......................... 0
+0:   contigious_checkpointing ........................ False
+0:   cpu_optimizer ................................... False
+0:   cpu_torch_adam .................................. False
+0:   curriculum_learning ............................. False
+0:   data_impl ....................................... mmap
+0:   data_parallel_size .............................. 64
+0:   data_path ....................................... None
+0:   dataloader_type ................................. single
+0:   DDP_impl ........................................ local
+0:   decoder_seq_length .............................. None
+0:   deepscale ....................................... False
+0:   deepscale_config ................................ None
+0:   deepspeed ....................................... True
+0:   deepspeed_activation_checkpointing .............. False
+0:   deepspeed_config ................................ ds_configs/2820936.json
+0:   deepspeed_mpi ................................... False
+0:   distribute_checkpointed_activations ............. False
+0:   distributed_backend ............................. nccl
+0:   embed_layernorm ................................. False
+0:   embedding_path .................................. None
+0:   encoder_seq_length .............................. 2048
+0:   eod_mask_loss ................................... False
+0:   eval_interval ................................... 1
+0:   eval_iters ...................................... 100
+0:   eval_only ....................................... True
+0:   evidence_data_path .............................. None
+0:   exit_duration_in_mins ........................... None
+0:   exit_interval ................................... None
+0:   ffn_hidden_size ................................. 7168
+0:   finetune ........................................ False
+0:   fp16 ............................................ False
+0:   fp16_lm_cross_entropy ........................... False
+0:   fp32_residual_connection ........................ False
+0:   gigaflos_no_embeds .............................. 0
+0:   global_batch_size ............................... 256
+0:   glu_activation .................................. None
+0:   hidden_dropout .................................. 0.1
+0:   hidden_size ..................................... 1792
+0:   hysteresis ...................................... 2
+0:   ict_head_size ................................... None
+0:   ict_load ........................................ None
+0:   img_dim ......................................... 224
+0:   indexer_batch_size .............................. 128
+0:   indexer_log_interval ............................ 1000
+0:   inference ....................................... False
+0:   init_method_std ................................. 0.02
+0:   init_method_xavier_uniform ...................... False
+0:   initial_loss_scale .............................. 4294967296
+0:   kill_switch_path ................................ kill-switch-1b11b51b5val
+0:   kv_channels ..................................... 128
+0:   layer_norm_fusion ............................... True
+0:   layernorm_epsilon ............................... 1e-05
+0:   lazy_mpu_init ................................... None
+0:   load ............................................ checkpoints_1b11b51b5
+0:   local_rank ...................................... None
+0:   log_batch_size_to_tensorboard ................... True
+0:   log_interval .................................... 10
+0:   log_learning_rate_to_tensorboard ................ True
+0:   log_level ....................................... None
+0:   log_level_replica ............................... None
+0:   log_loss_scale_to_tensorboard ................... True
+0:   log_num_zeros_in_grad ........................... False
+0:   log_params_norm ................................. False
+0:   log_path ........................................ None
+0:   log_timers_to_tensorboard ....................... True
+0:   log_validation_ppl_to_tensorboard ............... True
+0:   loss_on_targets_only ............................ False
+0:   loss_scale ...................................... 12.0
+0:   loss_scale_window ............................... 1000
+0:   lr .............................................. 0.0002
+0:   lr_decay_iters .................................. None
+0:   lr_decay_samples ................................ 1
+0:   lr_decay_style .................................. cosine
+0:   lr_decay_tokens ................................. None
+0:   lr_warmup_fraction .............................. None
+0:   lr_warmup_iters ................................. 0
+0:   lr_warmup_samples ............................... 0
+0:   make_vocab_size_divisible_by .................... 128
+0:   mask_prob ....................................... 0.15
+0:   masked_softmax_fusion ........................... True
+0:   max_position_embeddings ......................... 2048
+0:   mean_noise_span_length .......................... None
+0:   memory_centric_tiled_linear ..................... False
+0:   merge_file ...................................... gpt2/merges.txt
+0:   micro_batch_size ................................ 4
+0:   min_loss_scale .................................. 1.0
+0:   min_lr .......................................... 2e-05
+0:   mmap_warmup ..................................... False
+0:   no_load_optim ................................... True
+0:   no_load_rng ..................................... None
+0:   no_save_optim ................................... None
+0:   no_save_rng ..................................... None
+0:   noise_density ................................... None
+0:   num_attention_heads ............................. 14
+0:   num_channels .................................... 3
+0:   num_classes ..................................... 1000
+0:   num_layers ...................................... 26
+0:   num_layers_per_virtual_pipeline_stage ........... None
+0:   num_workers ..................................... 2
+0:   onnx_safe ....................................... None
+0:   openai_gelu ..................................... False
+0:   optimizer ....................................... adam
+0:   optimizer_fusion ................................ True
+0:   override_lr_scheduler ........................... True
+0:   pad_vocab_size_to ............................... None
+0:   params_dtype .................................... torch.bfloat16
+0:   partition_activations ........................... False
+0:   patch_dim ....................................... 16
+0:   pipeline_model_parallel_size .................... 1
+0:   position_embedding_type ......................... PositionEmbeddingType.absolute
+0:   pp_partition_method ............................. None
+0:   profile_backward ................................ False
+0:   query_in_block_prob ............................. 0.1
+0:   rampup_batch_size ............................... None
+0:   rank ............................................ 0
+0:   remote_device ................................... none
+0:   reset_attention_mask ............................ False
+0:   reset_position_ids .............................. False
+0:   reset_progress .................................. True
+0:   retriever_report_topk_accuracies ................ []
+0:   retriever_score_scaling ......................... False
+0:   retriever_seq_length ............................ 256
+0:   reweight_loss_based_on_position_frequency ....... False
+0:   sample_rate ..................................... 1.0
+0:   save ............................................ checkpoints_1b11b51b5
+0:   save_interval ................................... 1000
+0:   scatter_gather_tensors_in_pipeline .............. True
+0:   scattered_embeddings ............................ False
+0:   seed ............................................ 1234
+0:   seq_length ...................................... 2048
+0:   sgd_momentum .................................... 0.9
+0:   short_seq_prob .................................. 0.1
+0:   skip_train_iteration_range ...................... None
+0:   split ........................................... None
+0:   split_transformers .............................. False
+0:   sync_tp_duplicated_parameters ................... False
+0:   synchronize_each_layer .......................... False
+0:   tensor_model_parallel_size ...................... 1
+0:   tensorboard_dir ................................. tensorboard_1b11b51b5val
+0:   tensorboard_log_interval ........................ 1
+0:   tensorboard_queue_size .......................... 5
+0:   test_weighted_split_paths ....................... None
+0:   test_weighted_split_paths_path .................. None
+0:   tile_factor ..................................... 1
+0:   titles_data_path ................................ None
+0:   tokenizer_name_or_path .......................... None
+0:   tokenizer_type .................................. GPT2BPETokenizer
+0:   train_iters ..................................... None
+0:   train_samples ................................... 1
+0:   train_tokens .................................... None
+0:   train_weighted_split_names ...................... ['train']
+0:   train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']]
+0:   train_weighted_split_paths_path ................. None
+0:   train_weighted_split_splits ..................... [['0:1']]
+0:   train_weighted_split_weights .................... [['1.0']]
+0:   universal_checkpoint ............................ False
+0:   use_bnb_optimizer ............................... False
+0:   use_checkpoint_lr_scheduler ..................... False
+0:   use_contiguous_buffers_in_ddp ................... True
+0:   use_cpu_initialization .......................... None
+0:   use_one_sent_docs ............................... False
+0:   use_pin_memory .................................. False
+0:   valid_num_workers ............................... 2
+0:   valid_weighted_split_names ...................... ['validation']
+0:   valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']]
+0:   valid_weighted_split_paths_path ................. None
+0:   valid_weighted_split_splits ..................... [['0:1']]
+0:   valid_weighted_split_weights .................... [['1.0']]
+0:   virtual_pipeline_model_parallel_size ............ None
+0:   vocab_extra_ids ................................. 0
+0:   vocab_file ...................................... gpt2/vocab.json
+0:   weight_decay .................................... 0.1
+0:   world_size ...................................... 64
+0:   zero_allgather_bucket_size ...................... 0.0
+0:   zero_contigious_gradients ....................... False
+0:   zero_reduce_bucket_size ......................... 0.0
+0:   zero_reduce_scatter ............................. False
+0:   zero_stage ...................................... 0
+0: -------------------- end of arguments ---------------------
+0: setting number of micro-batches to constant 1
+0: > building GPT2BPETokenizer tokenizer ...
+0:  > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
+0: DeepSpeed general environment info:
+0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch']
+0: torch version .................... 1.13.0+rocm5.2
+0: torch cuda version ............... None
+0: torch hip version ................ 5.2.21151-afdc89f8
+0: nvcc version ..................... None
+0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed']
+0: deepspeed info ................... 0.7.5, unknown, unknown
+0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1
+0: **** Git info for Megatron: git_hash=unknown git_branch=unknown ****
+0: > initializing torch distributed ...
+0: [2023-02-09 23:02:42,490] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+7: > setting tensorboard ...
+0: > initializing tensor model parallel with size 1
+0: > initializing pipeline model parallel with size 1
+0: > setting random seeds to 1234 ...
+0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234
+0: > compiling dataset index builder ...
+0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data'
+0: make: Nothing to be done for 'default'.
+0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data'
+0: >>> done with dataset index builder. Compilation time: 0.106 seconds
+0: > compiling and loading fused kernels ...
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 87
+0: ninja: no work to do.
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 63
+0: ninja: no work to do.
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified]
+0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified]
+0: Total number of unsupported CUDA function calls: 0
+0: 
+0: 
+0: Total number of replaced kernel launches: 67
+0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/pfs/lustrep2/projappl/project_462000125/samantao-public/rocm/rocm-5.2.3/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so
+0: >>> done with compiling and loading fused kernels. Compilation time: 17.634 seconds
+0: time to initialize megatron (seconds): 7.064
+0: [after megatron is initialized] datetime: 2023-02-09 23:03:03 
+0: building GPT model ...
+0: [2023-02-09 23:03:03,212] [INFO] [utils.py:827:see_memory_usage] Before Building Model
+0: [2023-02-09 23:03:03,213] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB         Max_MA 0.0 GB         CA 0.0 GB         Max_CA 0 GB 
+0: [2023-02-09 23:03:03,213] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.11 GB, percent = 6.0%
+0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None
+0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=1, model=0): 1, ProcessCoord(pipe=0, data=2, model=0): 2, ProcessCoord(pipe=0, data=3, model=0): 3, ProcessCoord(pipe=0, data=4, model=0): 4, ProcessCoord(pipe=0, data=5, model=0): 5, ProcessCoord(pipe=0, data=6, model=0): 6, ProcessCoord(pipe=0, data=7, model=0): 7, ProcessCoord(pipe=0, data=8, model=0): 8, ProcessCoord(pipe=0, data=9, model=0): 9, ProcessCoord(pipe=0, data=10, model=0): 10, ProcessCoord(pipe=0, data=11, model=0): 11, ProcessCoord(pipe=0, data=12, model=0): 12, ProcessCoord(pipe=0, data=13, model=0): 13, ProcessCoord(pipe=0, data=14, model=0): 14, ProcessCoord(pipe=0, data=15, model=0): 15, ProcessCoord(pipe=0, data=16, model=0): 16, ProcessCoord(pipe=0, data=17, model=0): 17, ProcessCoord(pipe=0, data=18, model=0): 18, ProcessCoord(pipe=0, data=19, model=0): 19, ProcessCoord(pipe=0, data=20, model=0): 20, ProcessCoord(pipe=0, data=21, model=0): 21, ProcessCoord(pipe=0, data=22, model=0): 22, ProcessCoord(pi
+0: pe=0, data=23, model=0): 23, ProcessCoord(pipe=0, data=24, model=0): 24, ProcessCoord(pipe=0, data=25, model=0): 25, ProcessCoord(pipe=0, data=26, model=0): 26, ProcessCoord(pipe=0, data=27, model=0): 27, ProcessCoord(pipe=0, data=28, model=0): 28, ProcessCoord(pipe=0, data=29, model=0): 29, ProcessCoord(pipe=0, data=30, model=0): 30, ProcessCoord(pipe=0, data=31, model=0): 31, ProcessCoord(pipe=0, data=32, model=0): 32, ProcessCoord(pipe=0, data=33, model=0): 33, ProcessCoord(pipe=0, data=34, model=0): 34, ProcessCoord(pipe=0, data=35, model=0): 35, ProcessCoord(pipe=0, data=36, model=0): 36, ProcessCoord(pipe=0, data=37, model=0): 37, ProcessCoord(pipe=0, data=38, model=0): 38, ProcessCoord(pipe=0, data=39, model=0): 39, ProcessCoord(pipe=0, data=40, model=0): 40, ProcessCoord(pipe=0, data=41, model=0): 41, ProcessCoord(pipe=0, data=42, model=0): 42, ProcessCoord(pipe=0, data=43, model=0): 43, ProcessCoord(pipe=0, data=44, model=0): 44, ProcessCoord(pipe=0, data=45, model=0): 45, ProcessCoord(pipe=0, data=4
+0: 6, model=0): 46, ProcessCoord(pipe=0, data=47, model=0): 47, ProcessCoord(pipe=0, data=48, model=0): 48, ProcessCoord(pipe=0, data=49, model=0): 49, ProcessCoord(pipe=0, data=50, model=0): 50, ProcessCoord(pipe=0, data=51, model=0): 51, ProcessCoord(pipe=0, data=52, model=0): 52, ProcessCoord(pipe=0, data=53, model=0): 53, ProcessCoord(pipe=0, data=54, model=0): 54, ProcessCoord(pipe=0, data=55, model=0): 55, ProcessCoord(pipe=0, data=56, model=0): 56, ProcessCoord(pipe=0, data=57, model=0): 57, ProcessCoord(pipe=0, data=58, model=0): 58, ProcessCoord(pipe=0, data=59, model=0): 59, ProcessCoord(pipe=0, data=60, model=0): 60, ProcessCoord(pipe=0, data=61, model=0): 61, ProcessCoord(pipe=0, data=62, model=0): 62, ProcessCoord(pipe=0, data=63, model=0): 63}
+0: [2023-02-09 23:03:05,197] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer
+0: stage=0 layers=33
+0:      0: _to_float16
+0:      1: EmbeddingPipe
+0:      2: <lambda>
+0:      3: ParallelTransformerLayerPipe
+0:      4: ParallelTransformerLayerPipe
+0:      5: ParallelTransformerLayerPipe
+0:      6: ParallelTransformerLayerPipe
+0:      7: ParallelTransformerLayerPipe
+0:      8: ParallelTransformerLayerPipe
+0:      9: ParallelTransformerLayerPipe
+0:     10: ParallelTransformerLayerPipe
+0:     11: ParallelTransformerLayerPipe
+0:     12: ParallelTransformerLayerPipe
+0:     13: ParallelTransformerLayerPipe
+0:     14: ParallelTransformerLayerPipe
+0:     15: ParallelTransformerLayerPipe
+0:     16: ParallelTransformerLayerPipe
+0:     17: ParallelTransformerLayerPipe
+0:     18: ParallelTransformerLayerPipe
+0:     19: ParallelTransformerLayerPipe
+0:     20: ParallelTransformerLayerPipe
+0:     21: ParallelTransformerLayerPipe
+0:     22: ParallelTransformerLayerPipe
+0:     23: ParallelTransformerLayerPipe
+0:     24: ParallelTransformerLayerPipe
+0:     25: ParallelTransformerLayerPipe
+0:     26: ParallelTransformerLayerPipe
+0:     27: ParallelTransformerLayerPipe
+0:     28: ParallelTransformerLayerPipe
+0:     29: undo
+0:     30: MixedFusedLayerNorm
+0:     31: EmbeddingPipe
+0:     32: float16_to_fp32
+0:   loss: CrossEntropy
+0: [2023-02-09 23:03:05,484] [INFO] [utils.py:827:see_memory_usage] After Building Model
+0: [2023-02-09 23:03:05,485] [INFO] [utils.py:828:see_memory_usage] MA 2.05 GB         Max_MA 2.05 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2023-02-09 23:03:05,485] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.17 GB, percent = 6.0%
+0: setting training iterations to 0
+0: > learning rate decay style: cosine
+0: DeepSpeed is enabled.
+0: [2023-02-09 23:03:05,487] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown
+0: [2023-02-09 23:03:19,262] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False
+0: [2023-02-09 23:03:19,263] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer
+0: [2023-02-09 23:03:19,263] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer
+0: [2023-02-09 23:03:19,275] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam
+0: [2023-02-09 23:03:19,275] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer
+0: [2023-02-09 23:03:19,391] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer
+0: [2023-02-09 23:03:19,391] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB         Max_MA 2.06 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2023-02-09 23:03:19,392] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 30.84 GB, percent = 6.1%
+7: ninja: no work to do.
+7: Time to load utils op: 0.23356986045837402 seconds
+0: Time to load utils op: 0.10872554779052734 seconds
+0: Time to load utils op: 0.10894298553466797 seconds
+0: Time to load utils op: 0.1084299087524414 secondsTime to load utils op: 0.10905647277832031 seconds
+0: 
+0: Time to load utils op: 0.10917925834655762 secondsTime to load utils op: 0.10909605026245117 secondsTime to load utils op: 0.10839271545410156 seconds
+0: 
+0: 
+7: Time to load utils op: 0.10265994071960449 seconds
+7: Time to load utils op: 0.10144281387329102 seconds
+7: Time to load utils op: 0.1018972396850586 seconds
+7: Time to load utils op: 0.10172438621520996 seconds
+7: Time to load utils op: 0.10286927223205566 seconds
+7: Time to load utils op: 0.10248398780822754 secondsTime to load utils op: 0.10267233848571777 seconds
+7: 
+5: Time to load utils op: 0.10777807235717773 seconds
+5: Time to load utils op: 0.10777568817138672 seconds
+5: Time to load utils op: 0.10827493667602539 seconds
+5: Time to load utils op: 0.10802364349365234 seconds
+5: Time to load utils op: 0.10783147811889648 secondsTime to load utils op: 0.10817813873291016 seconds
+5: 
+5: Time to load utils op: 0.10782194137573242 seconds
+6: Time to load utils op: 0.10857582092285156 seconds
+6: Time to load utils op: 0.10863423347473145 secondsTime to load utils op: 0.10863161087036133 seconds
+6: 
+6: Time to load utils op: 0.10894608497619629 seconds
+6: Time to load utils op: 0.1085200309753418 seconds
+6: Time to load utils op: 0.10864448547363281 seconds
+6: Time to load utils op: 0.1086418628692627 seconds
+1: Time to load utils op: 0.1118626594543457 secondsTime to load utils op: 0.11190438270568848 seconds
+1: 
+1: Time to load utils op: 0.11191344261169434 secondsTime to load utils op: 0.11191129684448242 seconds
+1: 
+1: Time to load utils op: 0.11191797256469727 secondsTime to load utils op: 0.11191034317016602 seconds
+1: 
+1: Time to load utils op: 0.11191678047180176 seconds
+1: Time to load utils op: 0.11192870140075684 seconds
+2: Time to load utils op: 0.11117124557495117 seconds
+2: Time to load utils op: 0.11117911338806152 seconds
+2: Time to load utils op: 0.11118793487548828 seconds
+2: Time to load utils op: 0.11120963096618652 seconds
+2: Time to load utils op: 0.11122846603393555 seconds
+2: Time to load utils op: 0.11122512817382812 secondsTime to load utils op: 0.11121177673339844 seconds
+2: 
+2: Time to load utils op: 0.111236572265625 seconds
+3: Time to load utils op: 0.11118960380554199 secondsTime to load utils op: 0.11117959022521973 seconds
+3: 
+3: Time to load utils op: 0.11122345924377441 seconds
+3: Time to load utils op: 0.11122250556945801 seconds
+3: Time to load utils op: 0.11123418807983398 seconds
+3: Time to load utils op: 0.11122608184814453 secondsTime to load utils op: 0.11124157905578613 seconds
+3: 
+3: Time to load utils op: 0.1112527847290039 seconds
+4: Time to load utils op: 0.10989499092102051 secondsTime to load utils op: 0.10990285873413086 secondsTime to load utils op: 0.10990452766418457 secondsTime to load utils op: 0.10990333557128906 secondsTime to load utils op: 0.10990524291992188 seconds
+4: 
+4: 
+4: Time to load utils op: 0.10991334915161133 seconds
+4: 
+4: Time to load utils op: 0.10991358757019043 secondsTime to load utils op: 0.1099092960357666 seconds
+4: 
+4: 
+7: Time to load utils op: 0.0005280971527099609 seconds
+7: Time to load utils op: 0.0005304813385009766 seconds
+7: Time to load utils op: 0.0005471706390380859 seconds
+7: Time to load utils op: 0.000423431396484375 seconds
+7: Time to load utils op: 0.0005557537078857422 seconds
+7: Time to load utils op: 0.0006492137908935547 seconds
+7: Time to load utils op: 0.0006842613220214844 seconds
+7: Time to load utils op: 0.0004982948303222656 seconds
+0: Time to load utils op: 0.20360970497131348 seconds
+6: Time to load utils op: 0.3034961223602295 seconds
+5: Time to load utils op: 0.30406808853149414 seconds
+0: Time to load utils op: 0.0006165504455566406 seconds
+0: Time to load utils op: 0.0006113052368164062 seconds
+0: Time to load utils op: 0.0005795955657958984 seconds
+0: Time to load utils op: 0.0005714893341064453 seconds
+0: Time to load utils op: 0.0004494190216064453 seconds
+0: Time to load utils op: 0.0006427764892578125 seconds
+0: Time to load utils op: 0.0006692409515380859 seconds
+6: Time to load utils op: 0.00047016143798828125 seconds
+6: Time to load utils op: 0.0005619525909423828 seconds
+6: Time to load utils op: 0.0004703998565673828 secondsTime to load utils op: 0.0004954338073730469 seconds
+6: 
+6: Time to load utils op: 0.0004715919494628906 seconds
+6: Time to load utils op: 0.0004413127899169922 seconds
+6: Time to load utils op: 0.0004646778106689453 seconds
+6: Time to load utils op: 0.0004372596740722656 seconds
+5: Time to load utils op: 0.0005655288696289062 seconds
+5: Time to load utils op: 0.0005826950073242188 seconds
+5: Time to load utils op: 0.0006122589111328125 seconds
+5: Time to load utils op: 0.0006060600280761719 seconds
+5: Time to load utils op: 0.0006268024444580078 seconds
+5: Time to load utils op: 0.0006139278411865234 seconds
+5: Time to load utils op: 0.0005974769592285156 seconds
+5: Time to load utils op: 0.0006670951843261719 seconds
+4: Time to load utils op: 0.0009484291076660156 seconds
+1: Time to load utils op: 0.0009319782257080078 seconds
+1: Time to load utils op: 0.0008385181427001953 seconds
+1: Time to load utils op: 0.0008897781372070312 seconds
+4: Time to load utils op: 0.0010764598846435547 seconds
+4: Time to load utils op: 0.0012416839599609375 seconds
+1: Time to load utils op: 0.0011854171752929688 seconds
+1: Time to load utils op: 0.0011963844299316406 seconds
+2: Time to load utils op: 0.0011782646179199219 seconds
+1: Time to load utils op: 0.0011913776397705078 seconds
+1: Time to load utils op: 0.0011746883392333984 seconds
+2: Time to load utils op: 0.0011756420135498047 seconds
+1: Time to load utils op: 0.001245260238647461 seconds
+4: Time to load utils op: 0.0014786720275878906 seconds
+4: Time to load utils op: 0.0013957023620605469 seconds
+4: Time to load utils op: 0.0014011859893798828 seconds
+4: Time to load utils op: 0.0013840198516845703 seconds
+4: Time to load utils op: 0.0015354156494140625 seconds
+2: Time to load utils op: 0.0014214515686035156 seconds
+2: Time to load utils op: 0.0015032291412353516 secondsTime to load utils op: 0.0014638900756835938 seconds
+2: Time to load utils op: 0.0014421939849853516 seconds
+2: 
+2: Time to load utils op: 0.0014166831970214844 seconds
+2: Time to load utils op: 0.0014657974243164062 seconds
+3: Time to load utils op: 0.0007638931274414062 seconds
+3: Time to load utils op: 0.0009474754333496094 seconds
+3: Time to load utils op: 0.0008788108825683594 seconds
+3: Time to load utils op: 0.0010268688201904297 seconds
+3: Time to load utils op: 0.0009176731109619141 secondsTime to load utils op: 0.001085519790649414 seconds
+3: 
+3: Time to load utils op: 0.0010457038879394531 seconds
+3: Time to load utils op: 0.0010666847229003906 seconds
+0: [2023-02-09 23:03:19,721] [INFO] [utils.py:827:see_memory_usage] before initializing group 0
+0: [2023-02-09 23:03:19,722] [INFO] [utils.py:828:see_memory_usage] MA 2.04 GB         Max_MA 2.04 GB         CA 2.19 GB         Max_CA 2 GB 
+0: [2023-02-09 23:03:19,722] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:19,837] [INFO] [utils.py:827:see_memory_usage] after initializing group 0
+0: [2023-02-09 23:03:19,838] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB         Max_MA 4.24 GB         CA 5.44 GB         Max_CA 5 GB 
+0: [2023-02-09 23:03:19,838] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:19,940] [INFO] [utils.py:827:see_memory_usage] before initializing group 1
+0: [2023-02-09 23:03:19,940] [INFO] [utils.py:828:see_memory_usage] MA 4.24 GB         Max_MA 4.24 GB         CA 5.44 GB         Max_CA 5 GB 
+0: [2023-02-09 23:03:19,940] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,048] [INFO] [utils.py:827:see_memory_usage] after initializing group 1
+0: [2023-02-09 23:03:20,049] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,049] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,151] [INFO] [utils.py:827:see_memory_usage] before initializing group 2
+0: [2023-02-09 23:03:20,151] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,151] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,256] [INFO] [utils.py:827:see_memory_usage] after initializing group 2
+0: [2023-02-09 23:03:20,257] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,257] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,358] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer
+0: [2023-02-09 23:03:20,358] [INFO] [utils.py:828:see_memory_usage] MA 6.19 GB         Max_MA 6.19 GB         CA 8.31 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,358] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,466] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer
+0: [2023-02-09 23:03:20,467] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB         Max_MA 6.32 GB         CA 8.34 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,467] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,569] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer
+0: [2023-02-09 23:03:20,569] [INFO] [utils.py:828:see_memory_usage] MA 6.32 GB         Max_MA 6.32 GB         CA 8.34 GB         Max_CA 8 GB 
+0: [2023-02-09 23:03:20,569] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory:  used = 31.0 GB, percent = 6.2%
+0: [2023-02-09 23:03:20,570] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam
+0: [2023-02-09 23:03:20,570] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler
+0: [2023-02-09 23:03:20,570] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <megatron.learning_rates.AnnealingLR object at 0x14a9d921dfd0>
+0: [2023-02-09 23:03:20,570] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)]
+0: [2023-02-09 23:03:20,570] [INFO] [config.py:1007:print] DeepSpeedEngine configuration:
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   activation_checkpointing_config  {
+0:     "partition_activations": false, 
+0:     "contiguous_memory_optimization": false, 
+0:     "cpu_checkpointing": false, 
+0:     "number_checkpoints": null, 
+0:     "synchronize_checkpoint_boundary": false, 
+0:     "profile": false
+0: }
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   amp_enabled .................. False
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   amp_params ................... False
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   autotuning_config ............ {
+0:     "enabled": false, 
+0:     "start_step": null, 
+0:     "end_step": null, 
+0:     "metric_path": null, 
+0:     "arg_mappings": null, 
+0:     "metric": "throughput", 
+0:     "model_info": null, 
+0:     "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", 
+0:     "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", 
+0:     "overwrite": true, 
+0:     "fast": true, 
+0:     "start_profile_step": 3, 
+0:     "end_profile_step": 5, 
+0:     "tuner_type": "gridsearch", 
+0:     "tuner_early_stopping": 5, 
+0:     "tuner_num_trials": 50, 
+0:     "model_info_path": null, 
+0:     "mp_size": 1, 
+0:     "max_train_batch_size": null, 
+0:     "min_train_batch_size": 1, 
+0:     "max_train_micro_batch_size_per_gpu": 1.024000e+03, 
+0:     "min_train_micro_batch_size_per_gpu": 1, 
+0:     "num_tuning_micro_batch_sizes": 3
+0: }
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   bfloat16_enabled ............. True
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   checkpoint_parallel_write_pipeline  False
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   checkpoint_tag_validation_enabled  True
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   checkpoint_tag_validation_fail  False
+0: [2023-02-09 23:03:20,571] [INFO] [config.py:1011:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x14a8d91a4250>
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   communication_data_type ...... None
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa
+0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   curriculum_enabled ........... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   curriculum_params ............ False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   dataloader_drop_last ......... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   disable_allgather ............ False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   dump_state ................... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   dynamic_loss_scale_args ...... None
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_enabled ........... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_gas_boundary_resolution  1
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_layer_name ........ bert.encoder.layer
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_layer_num ......... 0
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_max_iter .......... 100
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_stability ......... 1e-06
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_tol ............... 0.01
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   eigenvalue_verbose ........... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   elasticity_enabled ........... False
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   flops_profiler_config ........ {
+0:     "enabled": false, 
+0:     "profile_step": 1, 
+0:     "module_depth": -1, 
+0:     "top_modules": 1, 
+0:     "detailed": true, 
+0:     "output_file": null
+0: }
+0: [2023-02-09 23:03:20,572] [INFO] [config.py:1011:print]   fp16_auto_cast ............... None
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   fp16_enabled ................. False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   fp16_master_weights_and_gradients  False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   global_rank .................. 0
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   gradient_accumulation_steps .. 1
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   gradient_clipping ............ 1.0
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   gradient_predivide_factor .... 1.0
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   initial_dynamic_scale ........ 1
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   load_universal_checkpoint .... False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   loss_scale ................... 1.0
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   memory_breakdown ............. False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x14a8d91a42e0>
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   nebula_config ................ {
+0:     "enabled": false, 
+0:     "persistent_storage_path": null, 
+0:     "persistent_time_interval": 100, 
+0:     "num_of_version_in_retention": 2, 
+0:     "enable_nebula_load": true, 
+0:     "load_path": null
+0: }
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   optimizer_legacy_fusion ...... False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   optimizer_name ............... None
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   optimizer_params ............. None
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   pld_enabled .................. False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   pld_params ................... False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   prescale_gradients ........... False
+0: [2023-02-09 23:03:20,573] [INFO] [config.py:1011:print]   scheduler_name ............... None
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   scheduler_params ............. None
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   sparse_attention ............. None
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   sparse_gradients_enabled ..... False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   steps_per_print .............. 2000
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   train_batch_size ............. 256
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   train_micro_batch_size_per_gpu  4
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   use_node_local_storage ....... False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   wall_clock_breakdown ......... False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   world_size ................... 64
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   zero_allow_untested_optimizer  False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   zero_enabled ................. False
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:1011:print]   zero_optimization_stage ...... 0
+0: [2023-02-09 23:03:20,574] [INFO] [config.py:996:print_user_config]   json = {
+0:     "train_micro_batch_size_per_gpu": 4, 
+0:     "train_batch_size": 256, 
+0:     "gradient_clipping": 1.0, 
+0:     "zero_optimization": {
+0:         "stage": 0
+0:     }, 
+0:     "bf16": {
+0:         "enabled": true
+0:     }, 
+0:     "steps_per_print": 2.000000e+03, 
+0:     "wall_clock_breakdown": false
+0: }
+0: Time to load utils op: 0.00047016143798828125 seconds
+0: [2023-02-09 23:03:20,575] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=4
+0: [2023-02-09 23:03:20,595] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=33 [0, 33) STAGE_PARAMS=1096338432 (1096.338M) TOTAL_PARAMS=1096338432 (1096.338M) UNIQUE_PARAMS=1096338432 (1096.338M)
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt...
+1: [2023-02-09 23:03:20,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/mp_rank_00_model_states.pt.
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:20,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:20,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:20,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:20,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+2: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+0: [2023-02-09 23:03:20,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+3: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+1: [2023-02-09 23:03:20,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+5: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+7: [2023-02-09 23:03:20,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+4: [2023-02-09 23:03:20,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_01-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_03-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_04-model_00-model_states.pt.
+2: [2023-02-09 23:03:21,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:21,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+4: [2023-02-09 23:03:21,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+5: [2023-02-09 23:03:21,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+5: [2023-02-09 23:03:21,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+4: [2023-02-09 23:03:21,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+7: [2023-02-09 23:03:21,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+0: [2023-02-09 23:03:21,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+7: [2023-02-09 23:03:21,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:21,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:21,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:21,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:21,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:21,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_05-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_06-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_07-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+1: [2023-02-09 23:03:22,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:22,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+4: [2023-02-09 23:03:22,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:22,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+3: [2023-02-09 23:03:22,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:22,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+6: [2023-02-09 23:03:22,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_08-model_00-model_states.pt.
+0: [2023-02-09 23:03:22,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:22,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:22,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:22,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:22,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:22,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:22,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:22,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:22,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_09-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+1: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+7: [2023-02-09 23:03:23,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+0: [2023-02-09 23:03:23,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+4: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:23,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:23,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+2: [2023-02-09 23:03:23,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+5: [2023-02-09 23:03:23,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+3: [2023-02-09 23:03:23,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:23,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:23,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:23,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:23,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_10-model_00-model_states.pt.
+6: [2023-02-09 23:03:23,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:23,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_11-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_12-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+2: [2023-02-09 23:03:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+1: [2023-02-09 23:03:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+0: [2023-02-09 23:03:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+5: [2023-02-09 23:03:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+3: [2023-02-09 23:03:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:24,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:24,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+0: [2023-02-09 23:03:24,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+6: [2023-02-09 23:03:24,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+4: [2023-02-09 23:03:24,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+6: [2023-02-09 23:03:24,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:24,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_13-model_00-model_states.pt.
+7: [2023-02-09 23:03:24,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_14-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_15-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_16-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+0: [2023-02-09 23:03:25,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+2: [2023-02-09 23:03:25,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+0: [2023-02-09 23:03:25,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt...
+3: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:25,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:25,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+3: [2023-02-09 23:03:25,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+1: [2023-02-09 23:03:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+6: [2023-02-09 23:03:25,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:25,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:25,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:25,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:25,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:25,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:25,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_17-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_18-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_19-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+2: [2023-02-09 23:03:26,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:26,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+4: [2023-02-09 23:03:26,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:26,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+0: [2023-02-09 23:03:26,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_20-model_00-model_states.pt.
+5: [2023-02-09 23:03:26,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:26,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:26,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:26,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:26,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:26,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_21-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_22-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_23-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+5: [2023-02-09 23:03:27,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:27,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+1: [2023-02-09 23:03:27,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:27,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+6: [2023-02-09 23:03:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+2: [2023-02-09 23:03:27,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+6: [2023-02-09 23:03:27,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+0: [2023-02-09 23:03:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+7: [2023-02-09 23:03:27,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:27,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+4: [2023-02-09 23:03:27,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_24-model_00-model_states.pt.
+3: [2023-02-09 23:03:27,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:27,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:27,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+2: [2023-02-09 23:03:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+5: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+1: [2023-02-09 23:03:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+0: [2023-02-09 23:03:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+6: [2023-02-09 23:03:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:30,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:30,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:30,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+3: [2023-02-09 23:03:30,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+7: [2023-02-09 23:03:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_25-model_00-model_states.pt.
+4: [2023-02-09 23:03:30,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:30,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_26-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_27-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt...
+3: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt...
+3: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt...
+3: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt...
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+7: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+3: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt...
+3: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+3: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt...
+6: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt...
+1: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+1: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0:  > overriding learning rate value to 0.0002
+0:  > overriding minimum learning rate value to 2e-05
+0:  > overriding warmup iterations value to 0
+0:  > overriding total number of iterations value to 1
+0:  > overriding decay style value to cosine
+4: [2023-02-09 23:03:31,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt...
+6: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+1: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_28-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+2: [2023-02-09 23:03:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+6: [2023-02-09 23:03:31,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+2: [2023-02-09 23:03:31,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+6: [2023-02-09 23:03:31,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt...
+6: [2023-02-09 23:03:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt...
+4: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+0: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt...
+5: [2023-02-09 23:03:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+5: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt...
+4: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+0: [2023-02-09 23:03:31,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/layer_30-model_00-model_states.pt.
+4: [2023-02-09 23:03:31,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt...
+0: [2023-02-09 23:03:31,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt...
+2: [2023-02-09 23:03:32,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 20
+2: [2023-02-09 23:03:32,348] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 20
+0: [2023-02-09 23:03:32,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 6
+7: [2023-02-09 23:03:32,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,367] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 60
+0: [2023-02-09 23:03:32,371] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 6
+7: [2023-02-09 23:03:32,374] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 60
+4: [2023-02-09 23:03:32,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,381] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 33
+4: [2023-02-09 23:03:32,387] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 33
+7: [2023-02-09 23:03:32,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,403] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 59
+7: [2023-02-09 23:03:32,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 59
+3: [2023-02-09 23:03:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,417] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 26
+3: [2023-02-09 23:03:32,423] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 26
+1: [2023-02-09 23:03:32,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,431] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 10
+1: [2023-02-09 23:03:32,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 10
+6: [2023-02-09 23:03:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,439] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 50
+6: [2023-02-09 23:03:32,449] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 50
+4: [2023-02-09 23:03:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,470] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 36
+7: [2023-02-09 23:03:32,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 57
+4: [2023-02-09 23:03:32,476] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 36
+7: [2023-02-09 23:03:32,481] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 57
+6: [2023-02-09 23:03:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,489] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 48
+6: [2023-02-09 23:03:32,489] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 51
+4: [2023-02-09 23:03:32,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,495] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 38
+3: [2023-02-09 23:03:32,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,497] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 25
+6: [2023-02-09 23:03:32,498] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 48
+6: [2023-02-09 23:03:32,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 51
+4: [2023-02-09 23:03:32,501] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 38
+3: [2023-02-09 23:03:32,504] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 25
+6: [2023-02-09 23:03:32,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 53
+1: [2023-02-09 23:03:32,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,504] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 11
+6: [2023-02-09 23:03:32,510] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 53
+1: [2023-02-09 23:03:32,511] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 11
+0: [2023-02-09 23:03:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,512] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 3
+1: [2023-02-09 23:03:32,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,515] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 9
+7: [2023-02-09 23:03:32,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,517] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 62
+0: [2023-02-09 23:03:32,518] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 3
+5: [2023-02-09 23:03:32,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,520] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 46
+1: [2023-02-09 23:03:32,521] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 9
+7: [2023-02-09 23:03:32,524] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 62
+4: [2023-02-09 23:03:32,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 39
+5: [2023-02-09 23:03:32,526] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 46
+4: [2023-02-09 23:03:32,533] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 39
+4: [2023-02-09 23:03:32,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,541] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 34
+0: [2023-02-09 23:03:32,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,545] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 2
+3: [2023-02-09 23:03:32,545] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 27
+2: [2023-02-09 23:03:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,546] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 19
+7: [2023-02-09 23:03:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 61
+4: [2023-02-09 23:03:32,548] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 34
+3: [2023-02-09 23:03:32,552] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 27
+2: [2023-02-09 23:03:32,552] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 19
+0: [2023-02-09 23:03:32,553] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 2
+0: [2023-02-09 23:03:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 0
+7: [2023-02-09 23:03:32,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 61
+1: [2023-02-09 23:03:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,556] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 14
+1: [2023-02-09 23:03:32,562] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 14
+6: [2023-02-09 23:03:32,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 52
+0: [2023-02-09 23:03:32,563] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 0
+3: [2023-02-09 23:03:32,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,563] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 24
+0: could not find arguments in the checkpoint ...
+0:  checkpoint version 3.0
+3: [2023-02-09 23:03:32,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 24
+6: [2023-02-09 23:03:32,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 52
+2: [2023-02-09 23:03:32,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 23
+4: [2023-02-09 23:03:32,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,574] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 32
+7: [2023-02-09 23:03:32,575] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 58
+2: [2023-02-09 23:03:32,577] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 23
+5: [2023-02-09 23:03:32,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,579] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 43
+4: [2023-02-09 23:03:32,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 32
+7: [2023-02-09 23:03:32,582] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 58
+5: [2023-02-09 23:03:32,586] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 43
+4: [2023-02-09 23:03:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 35
+0: [2023-02-09 23:03:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,591] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 4
+6: [2023-02-09 23:03:32,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,592] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 54
+4: [2023-02-09 23:03:32,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 35
+7: [2023-02-09 23:03:32,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 54
+7: [2023-02-09 23:03:32,599] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 63
+7: [2023-02-09 23:03:32,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt.
+7: [2023-02-09 23:03:32,599] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 56
+0: [2023-02-09 23:03:32,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 4
+5: [2023-02-09 23:03:32,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,601] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 40
+6: [2023-02-09 23:03:32,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,603] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 49
+7: [2023-02-09 23:03:32,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 56
+7: [2023-02-09 23:03:32,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 63
+5: [2023-02-09 23:03:32,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 40
+6: [2023-02-09 23:03:32,609] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 49
+3: [2023-02-09 23:03:32,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,622] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 30
+2: [2023-02-09 23:03:32,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,622] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 21
+5: [2023-02-09 23:03:32,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,624] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 44
+1: [2023-02-09 23:03:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,629] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 13
+2: [2023-02-09 23:03:32,628] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 21
+5: [2023-02-09 23:03:32,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,629] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 45
+3: [2023-02-09 23:03:32,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 30
+2: [2023-02-09 23:03:32,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,630] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 22
+5: [2023-02-09 23:03:32,631] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 44
+1: [2023-02-09 23:03:32,635] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 13
+5: [2023-02-09 23:03:32,636] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 45
+2: [2023-02-09 23:03:32,636] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 22
+3: [2023-02-09 23:03:32,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,646] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 28
+3: [2023-02-09 23:03:32,654] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 28
+3: [2023-02-09 23:03:32,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,670] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 31
+3: [2023-02-09 23:03:32,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt.
+3: [2023-02-09 23:03:32,671] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 29
+0: [2023-02-09 23:03:32,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,673] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 5
+5: [2023-02-09 23:03:32,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,674] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 41
+3: [2023-02-09 23:03:32,677] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 31
+3: [2023-02-09 23:03:32,679] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 29
+0: [2023-02-09 23:03:32,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 5
+5: [2023-02-09 23:03:32,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 41
+1: [2023-02-09 23:03:32,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 12
+2: [2023-02-09 23:03:32,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,694] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 18
+1: [2023-02-09 23:03:32,696] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 12
+6: [2023-02-09 23:03:32,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt.
+6: [2023-02-09 23:03:32,697] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 55
+5: [2023-02-09 23:03:32,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,699] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 42
+2: [2023-02-09 23:03:32,701] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 18
+4: [2023-02-09 23:03:32,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt.
+4: [2023-02-09 23:03:32,701] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 37
+1: [2023-02-09 23:03:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,702] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 8
+5: [2023-02-09 23:03:32,706] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 42
+6: [2023-02-09 23:03:32,706] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 55
+4: [2023-02-09 23:03:32,707] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 37
+1: [2023-02-09 23:03:32,710] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 8
+5: [2023-02-09 23:03:32,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt.
+5: [2023-02-09 23:03:32,718] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 47
+5: [2023-02-09 23:03:32,725] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 47
+0: [2023-02-09 23:03:32,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:32,743] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 7
+0: [2023-02-09 23:03:32,751] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 7
+1: [2023-02-09 23:03:32,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt.
+1: [2023-02-09 23:03:32,783] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 15
+1: [2023-02-09 23:03:32,790] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 15
+2: [2023-02-09 23:03:32,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:32,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 17
+2: [2023-02-09 23:03:32,806] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 17
+0: [2023-02-09 23:03:36,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt.
+0: [2023-02-09 23:03:36,197] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 1
+0: [2023-02-09 23:03:36,205] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 1
+2: [2023-02-09 23:03:36,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from checkpoints_1b11b51b5/global_step2891/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt.
+2: [2023-02-09 23:03:36,209] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 64 ZeRO state_dicts for rank 16
+2: [2023-02-09 23:03:36,215] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 64 zero partition checkpoints for rank 16
+0:   successfully loaded checkpoint from checkpoints_1b11b51b5 at iteration 0
+7: time (ms) | load-checkpoint: 15636.40
+0: estimated model parameters: 1.096338432
+0: estimated model parameters without embeddings: 1.002523648
+0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-09 23:03:36 
+0: > building train, validation, and test datasets ...
+0:  > datasets target sizes (minimum size):
+0:     train:      1
+0:     validation: 25600
+0:     test:       25600
+0: > building train, validation, and test datasets for GPT ...
+0:  > building dataset index ...
+0:     reading sizes...
+0:     reading pointers...
+0:     reading document index...
+0:     creating numpy buffer of mmap...
+0:     creating memory view of numpy buffer...
+0:  > finished creating indexed dataset in 0.031559 seconds
+0:     number of documents: 3133972
+0:  > dataset split:
+0:     train:
+0:      document indices in [0, 3133972) total of 3133972 documents
+0:  > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy
+0:  > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy
+0:  > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy
+0:     loaded indexed file in 0.111 seconds
+0:     total number of samples: 731002
+0:     total number of epochs: 1
+0:  > building dataset index ...
+0:     reading sizes...
+0:     reading pointers...
+0:     reading document index...
+0:     creating numpy buffer of mmap...
+0:     creating memory view of numpy buffer...
+0:  > finished creating indexed dataset in 0.026850 seconds
+0:     number of documents: 364608
+0:  > dataset split:
+0:     validation:
+0:      document indices in [0, 364608) total of 364608 documents
+0:  > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_doc_idx.npy
+0:  > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_sample_idx.npy
+0:  > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_25600ns_2048sl_1234s_shuffle_idx.npy
+0:     loaded indexed file in 0.059 seconds
+0:     total number of samples: 84978
+0:     total number of epochs: 1
+0: > finished creating GPT datasets ...
+0: [after dataloaders are built] datetime: 2023-02-09 23:03:48 
+0: done with setup ...
+0: training ...
+7: time (ms) | model-and-optimizer-setup: 33354.79 | train/valid/test-data-iterators-setup: 12163.25
+0: [after training is done] datetime: 2023-02-09 23:03:48 
+7: -----------------------------------------------------------------------------------------------------------------
+7: validation loss at the end of training for val data | lm loss value: 3.680017E+00 | lm loss PPL: 3.964708E+01 | 
+7: -----------------------------------------------------------------------------------------------------------------
+END 2820936: Thu 09 Feb 2023 11:04:36 PM EET
diff --git a/1b11b51b5/sbatch_1b11b5pile.sh b/1b11b51b5/sbatch_1b11b5pile.sh
new file mode 100755
index 0000000000000000000000000000000000000000..04d117d3e2dd892615860085722b8ca31ed3ee08
--- /dev/null
+++ b/1b11b51b5/sbatch_1b11b5pile.sh
@@ -0,0 +1,162 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=8
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=32
+#SBATCH --mem=256G
+#SBATCH -p eap
+#SBATCH -t 12:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=1b1
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=checkpoints_$VARIANT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+# Start from scratch
+rm -rf "$CHECKPOINT_PATH" "$TENSORBOARD_PATH"
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+
+PP_SIZE=1
+TP_SIZE=1
+
+MICRO_BATCH_SIZE=4
+GRADIENT_ACCUMULATION_STEPS=1
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_1143M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 1516071000
+# -> Samples: 740269
+TRAIN_SAMPLES=740_269
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 7403 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --loss-scale 12 \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1000 \
+    --eval-iters 1 \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --data-path $DATA_PATH \
+    --data-impl mmap \
+    --split 949,50,1 \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"
diff --git a/1b11b51b5/tensorboard/1b11b51b5pile/events.out.tfevents.1669314189.nid005048.20312.0 b/1b11b51b5/tensorboard/1b11b51b5pile/events.out.tfevents.1669314189.nid005048.20312.0
new file mode 100644
index 0000000000000000000000000000000000000000..0e72ded09e670617fdaf5a63bfc1e8fe21a74ee6
--- /dev/null
+++ b/1b11b51b5/tensorboard/1b11b51b5pile/events.out.tfevents.1669314189.nid005048.20312.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a2d9666bf4f9f0583753dc02bb4b9f346a705315f4dfc6eaadb7ca741b2b2b9
+size 5154224
diff --git a/1b11b51b5/tensorboard/lm1-1b1-1b5-inv-8e-4/2074617/events.out.tfevents.1669478384.nid005099.75915.0 b/1b11b51b5/tensorboard/lm1-1b1-1b5-inv-8e-4/2074617/events.out.tfevents.1669478384.nid005099.75915.0
new file mode 100644
index 0000000000000000000000000000000000000000..2c896fc15e14842fbef98366c4fdd93776842715
--- /dev/null
+++ b/1b11b51b5/tensorboard/lm1-1b1-1b5-inv-8e-4/2074617/events.out.tfevents.1669478384.nid005099.75915.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350d7321c4a1dd67468bebeeadfea050a1e77856681903e305279cd474dd26f8
+size 5154554
diff --git a/1b11b51b5/tensorboard/lm1-1b1-1b5-inv/2074481/events.out.tfevents.1669473795.nid005111.98262.0 b/1b11b51b5/tensorboard/lm1-1b1-1b5-inv/2074481/events.out.tfevents.1669473795.nid005111.98262.0
new file mode 100644
index 0000000000000000000000000000000000000000..6098c770d8ea923bf0ad2bc15cb89300b4684e5c
--- /dev/null
+++ b/1b11b51b5/tensorboard/lm1-1b1-1b5-inv/2074481/events.out.tfevents.1669473795.nid005111.98262.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4392c5a7a0b7d37bb7da44eecd0601c1fd5f65a4960ce1a576755cc45a0f4cfa
+size 5154539
diff --git a/1b11b51b5/transformers/config.json b/1b11b51b5/transformers/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..7120acb75187622c061f0425674b658dd9155502
--- /dev/null
+++ b/1b11b51b5/transformers/config.json
@@ -0,0 +1 @@
+{"vocab_size": 50304, "n_positions": 2048, "n_embd": 1792, "n_layer": 26, "n_head": 14, "n_inner": 7168, "activation_function": "gelu", "resid_pdrop": 0.1, "embd_pdrop": 0.1, "attn_pdrop": 0.1, "layer_norm_epsilon": 1e-05, "initializer_range": 0.02, "summary_type": "cls_index", "summary_use_proj": true, "summary_activation": null, "summary_first_dropout": 0.1, "summary_proj_to_labels": true, "scale_attn_weights": true, "use_cache": true, "scale_attn_by_inverse_layer_idx": false, "reorder_and_upcast_attn": false, "bos_token_id": 50256, "eos_token_id": 50256, "return_dict": true, "output_hidden_states": false, "output_attentions": false, "torchscript": false, "torch_dtype": null, "use_bfloat16": false, "tf_legacy_loss": false, "pruned_heads": {}, "tie_word_embeddings": true, "is_encoder_decoder": false, "is_decoder": false, "cross_attention_hidden_size": null, "add_cross_attention": false, "tie_encoder_decoder": false, "max_length": 20, "min_length": 0, "do_sample": false, "early_stopping": false, "num_beams": 1, "num_beam_groups": 1, "diversity_penalty": 0.0, "temperature": 1.0, "top_k": 50, "top_p": 1.0, "typical_p": 1.0, "repetition_penalty": 1.0, "length_penalty": 1.0, "no_repeat_ngram_size": 0, "encoder_no_repeat_ngram_size": 0, "bad_words_ids": null, "num_return_sequences": 1, "chunk_size_feed_forward": 0, "output_scores": false, "return_dict_in_generate": false, "forced_bos_token_id": null, "forced_eos_token_id": null, "remove_invalid_values": false, "exponential_decay_length_penalty": null, "suppress_tokens": null, "begin_suppress_tokens": null, "architectures": ["GPT2LMHeadModel"], "finetuning_task": null, "id2label": {"0": "LABEL_0", "1": "LABEL_1"}, "label2id": {"LABEL_0": 0, "LABEL_1": 1}, "tokenizer_class": null, "prefix": null, "pad_token_id": null, "sep_token_id": null, "decoder_start_token_id": null, "task_specific_params": null, "problem_type": null, "_name_or_path": "", "transformers_version": "4.25.0.dev0", "n_ctx": 1024, "gradient_checkpointing": false, "model_type": "gpt2"}
\ No newline at end of file
diff --git a/1b11b51b5/transformers/pytorch_model.bin b/1b11b51b5/transformers/pytorch_model.bin
new file mode 100644
index 0000000000000000000000000000000000000000..80772b1ca598bce6d19ec8308e2ffad03513e8c0
--- /dev/null
+++ b/1b11b51b5/transformers/pytorch_model.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f4287bc4afae8d67b61b2e4b0ede3d8769f0c02f4b15e2f2691c85c11e353b
+size 2410895501