20241212

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+loss.png filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "assets/facebookai_xlm_roberta_base_pretrain_20240912_e1",
+  "_name_or_path": "assets/facebookai_xlm_roberta_base",
   "architectures": [
     "XLMRobertaForMaskedLM"
   ],
@@ -21,7 +21,7 @@
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.44.1",
+  "transformers_version": "4.47.0",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 250002

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91c427bad7ea8cefe0c0a798a343dd9704e99141bb36759f589d2d42839cda57
+oid sha256:98afd8269c67540955c75445024b3e9f34640a3d60f70c5e7cec2f804e829877
 size 1113205088

special_tokens_map.json

@@ -1,25 +1,7 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "cls_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -27,25 +9,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sep_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -42,10 +42,11 @@
     }
   },
   "bos_token": "<s>",
-  "clean_up_tokenization_spaces": true,
+  "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "do_lower_case": false,
   "eos_token": "</s>",
+  "extra_special_tokens": {},
   "mask_token": "<mask>",
   "model_max_length": 512,
   "pad_token": "<pad>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0348346af01c92edf289dac4416c73eb11e0ed0a856d90276465524cf1a2c7b
-size 5176
+oid sha256:b08346156c4ed671d8256f9ee9e056de5bf583dbb21d4361955304edf918c897
+size 5304

training_args.json

@@ -1,5 +1,5 @@
 {
-    "output_dir": "output/facebookai_xlm_roberta_base_pretrain_20240912_e1_pretrain",
+    "output_dir": "output/facebookai_xlm_roberta_base_pretrain",
     "overwrite_output_dir": false,
     "do_train": false,
     "do_eval": false,
@@ -14,13 +14,13 @@
     "eval_accumulation_steps": null,
     "eval_delay": 0,
     "torch_empty_cache_steps": null,
-    "learning_rate": 1e-05,
+    "learning_rate": 2e-05,
     "weight_decay": 0.01,
     "adam_beta1": 0.9,
     "adam_beta2": 0.999,
     "adam_epsilon": 1e-08,
     "max_grad_norm": 1.0,
-    "num_train_epochs": 1,
+    "num_train_epochs": 2,
     "max_steps": -1,
     "lr_scheduler_type": "linear",
     "lr_scheduler_kwargs": {},
@@ -32,10 +32,10 @@
     "logging_dir": "logs",
     "logging_strategy": "steps",
     "logging_first_step": false,
-    "logging_steps": 30,
+    "logging_steps": 10,
     "logging_nan_inf_filter": true,
     "save_strategy": "steps",
-    "save_steps": 300,
+    "save_steps": 100,
     "save_total_limit": 3,
     "save_safetensors": true,
     "save_on_each_node": false,
@@ -65,7 +65,7 @@
     "dataloader_num_workers": 0,
     "dataloader_prefetch_factor": null,
     "past_index": -1,
-    "run_name": "output/facebookai_xlm_roberta_base_pretrain_20240912_e1_pretrain",
+    "run_name": "output/facebookai_xlm_roberta_base_pretrain",
     "disable_tqdm": false,
     "remove_unused_columns": true,
     "label_names": null,
@@ -92,13 +92,12 @@
     },
     "deepspeed": null,
     "label_smoothing_factor": 0.0,
-    "optim": "adamw_8bit",
+    "optim": "ademamix_8bit",
     "optim_args": null,
     "adafactor": false,
     "group_by_length": false,
     "length_column_name": "length",
     "report_to": [
-        "tensorboard",
         "wandb"
     ],
     "ddp_find_unused_parameters": null,
@@ -113,11 +112,12 @@
     "hub_model_id": null,
     "hub_strategy": "every_save",
     "hub_token": "<HUB_TOKEN>",
-    "hub_private_repo": false,
+    "hub_private_repo": null,
     "hub_always_push": false,
     "gradient_checkpointing": false,
     "gradient_checkpointing_kwargs": null,
     "include_inputs_for_metrics": false,
+    "include_for_metrics": [],
     "eval_do_concat_batches": true,
     "fp16_backend": "auto",
     "evaluation_strategy": null,
@@ -141,5 +141,7 @@
     "optim_target_modules": null,
     "batch_eval_metrics": false,
     "eval_on_start": false,
-    "eval_use_gather_object": false
+    "use_liger_kernel": false,
+    "eval_use_gather_object": false,
+    "average_tokens_across_devices": false
 }