Training in progress, epoch 1

ipu_config.json

@@ -7,7 +7,7 @@
   "execute_encoder_on_cpu_for_generation": false,
   "gradient_accumulation_steps": 16,
   "inference_device_iterations": 5,
-  "inference_replication_factor": 1,
+  "inference_replication_factor": 4,
   "ipus_per_replica": 4,
   "layers_per_ipu": [
     0,
@@ -22,8 +22,8 @@
   "optimum_version": "1.5.1",
   "output_mode": "final",
   "recompute_checkpoint_every_layer": true,
-  "replicated_tensor_sharding": false,
-  "replication_factor": 1,
+  "replicated_tensor_sharding": true,
+  "replication_factor": 4,
   "seed": 42,
   "sharded_execution_for_inference": false,
   "transformers_version": "4.20.1"

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e773a98956d05c3fc38808129af94edde10bb5a8ff5bb0a1f31393937c046439
+oid sha256:9ed051f6ed1cf87b40cf93b20e1b0e3f3145d554fcb685884a60589355f13cd9
 size 249363181

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb8193e8397e023d4ae81c908969d3cf7d6b5b7da1d85c4bba54161e9837e798
+oid sha256:a3ed0e2087b1c2aa3cb106e60d7e10ffd7367824a41542bcf441919ca3d0677d
 size 2735