don't add position_ids for evals (#591)
Browse files
src/axolotl/utils/trainer.py
CHANGED
@@ -403,10 +403,11 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
|
|
403 |
|
404 |
if cfg.sample_packing:
|
405 |
train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
|
406 |
-
if
|
407 |
-
|
408 |
-
|
409 |
-
|
|
|
410 |
return train_dataset, eval_dataset
|
411 |
|
412 |
|
|
|
403 |
|
404 |
if cfg.sample_packing:
|
405 |
train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
|
406 |
+
if cfg.eval_sample_packing is not False:
|
407 |
+
if eval_dataset:
|
408 |
+
eval_dataset = eval_dataset.map(
|
409 |
+
add_position_ids, num_proc=os.cpu_count()
|
410 |
+
)
|
411 |
return train_dataset, eval_dataset
|
412 |
|
413 |
|