Delete llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/added_tokens.json

@@ -1,6 +0,0 @@
-{
-  "<image>": 151646,
-  "<|endoftext|>": 151643,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/config.json

@@ -1,221 +0,0 @@
-{
-  "_name_or_path": "lmms-lab/llava-onevision-qwen2-7b-si",
-  "add_faster_video": false,
-  "add_time_instruction": true,
-  "architectures": [
-    "LlavaQwenForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151645,
-  "faster_token_stride": 10,
-  "force_sample": true,
-  "hidden_act": "silu",
-  "hidden_size": 3584,
-  "ignore_index": -100,
-  "image_aspect_ratio": "anyres_max_9",
-  "image_crop_resolution": null,
-  "image_grid_pinpoints": [
-    [
-      384,
-      384
-    ],
-    [
-      384,
-      768
-    ],
-    [
-      384,
-      1152
-    ],
-    [
-      384,
-      1536
-    ],
-    [
-      384,
-      1920
-    ],
-    [
-      384,
-      2304
-    ],
-    [
-      768,
-      384
-    ],
-    [
-      768,
-      768
-    ],
-    [
-      768,
-      1152
-    ],
-    [
-      768,
-      1536
-    ],
-    [
-      768,
-      1920
-    ],
-    [
-      768,
-      2304
-    ],
-    [
-      1152,
-      384
-    ],
-    [
-      1152,
-      768
-    ],
-    [
-      1152,
-      1152
-    ],
-    [
-      1152,
-      1536
-    ],
-    [
-      1152,
-      1920
-    ],
-    [
-      1152,
-      2304
-    ],
-    [
-      1536,
-      384
-    ],
-    [
-      1536,
-      768
-    ],
-    [
-      1536,
-      1152
-    ],
-    [
-      1536,
-      1536
-    ],
-    [
-      1536,
-      1920
-    ],
-    [
-      1536,
-      2304
-    ],
-    [
-      1920,
-      384
-    ],
-    [
-      1920,
-      768
-    ],
-    [
-      1920,
-      1152
-    ],
-    [
-      1920,
-      1536
-    ],
-    [
-      1920,
-      1920
-    ],
-    [
-      1920,
-      2304
-    ],
-    [
-      2304,
-      384
-    ],
-    [
-      2304,
-      768
-    ],
-    [
-      2304,
-      1152
-    ],
-    [
-      2304,
-      1536
-    ],
-    [
-      2304,
-      1920
-    ],
-    [
-      2304,
-      2304
-    ]
-  ],
-  "image_split_resolution": null,
-  "image_token_index": 151646,
-  "initializer_range": 0.02,
-  "intermediate_size": 18944,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 28,
-  "mm_hidden_size": 1152,
-  "mm_newline_position": "grid",
-  "mm_patch_merge_type": "spatial_unpad",
-  "mm_projector_lr": null,
-  "mm_projector_type": "mlp2x_gelu",
-  "mm_resampler_type": null,
-  "mm_spatial_pool_mode": "bilinear",
-  "mm_spatial_pool_stride": 2,
-  "mm_tunable_parts": "mm_vision_tower,mm_mlp_adapter,mm_language_model",
-  "mm_use_im_patch_token": false,
-  "mm_use_im_start_end": false,
-  "mm_vision_select_feature": "patch",
-  "mm_vision_select_layer": -2,
-  "mm_vision_tower": "google/siglip-so400m-patch14-384",
-  "mm_vision_tower_lr": 2e-06,
-  "model_type": "llava",
-  "num_attention_heads": 28,
-  "num_hidden_layers": 28,
-  "num_key_value_heads": 4,
-  "pos_skipping_range": 4096,
-  "projector_hidden_act": "gelu",
-  "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
-  "rope_theta": 1000000.0,
-  "sliding_window": 131072,
-  "text_config": {
-    "model_type": "llama"
-  },
-  "tie_word_embeddings": false,
-  "tokenizer_model_max_length": 32768,
-  "tokenizer_padding_side": "right",
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0.dev0",
-  "use_cache": false,
-  "use_mm_proj": true,
-  "use_pos_skipping": false,
-  "use_sliding_window": false,
-  "vision_config": {
-    "hidden_size": 1024,
-    "image_size": 336,
-    "intermediate_size": 4096,
-    "model_type": "clip_vision_model",
-    "num_attention_heads": 16,
-    "num_hidden_layers": 24,
-    "patch_size": 14,
-    "projection_dim": 768,
-    "vocab_size": 32000
-  },
-  "vision_feature_layer": -2,
-  "vision_feature_select_strategy": "default",
-  "vision_tower_pretrained": null
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/generation_config.json

@@ -1,14 +0,0 @@
-{
-  "bos_token_id": 151643,
-  "do_sample": true,
-  "eos_token_id": [
-    151645,
-    151643
-  ],
-  "pad_token_id": 151643,
-  "repetition_penalty": 1.05,
-  "temperature": 0.7,
-  "top_k": 20,
-  "top_p": 0.8,
-  "transformers_version": "4.40.0.dev0"
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fb0414e5cc5c727ac7ca93dd58b4498d1295c378288583c133422bf2bf8b2b03
-size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0b1bfe6f71325ca71e2dce3060023ce741018771fcce67d70ce47b8872481e75
-size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:79e3a8875331146d1454f778947d829dd7a51ef376fe87054b3d366a96747695
-size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7995817f58750dcacb8b4b68d03bf6c82db08a1d1402e03a0f6e3172b1155cb0
-size 24091055846

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2d3009eb1490aef3931e316ec63030a529944f011418a919367481c98888d618
-size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b78406943c6fb485f8fae89e4291616ce089eaeeae56cc70d671077b60c3fb4c
-size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5406035a1c98d1ca12650b7d97bba5aa5724d60235e9ca6a4c70c185bfa5d138
-size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3eadc5a1eccc5215acda5b04ec77e33be02e6c3545d924d9d0d64b1f37783ae2
-size 419351

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/latest

@@ -1 +0,0 @@
-global_step500

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00001-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f966ae2c87927d9413e33e504a7bb8e850655b5250ae7d66f27063d91c914448
-size 4877668032

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00002-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:03cf039de0fb75cfaabd873f824eb7136d99bcb350175c80ce77e2da396e6f55
-size 4932751008

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00003-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e4ab349e68cbecd2818266f4cbdedbd5a3fc0182f5c0ba70a693ea0441c5f9de
-size 4994571904

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model-00004-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:11d909a1044aa3465efa12c5ce42d521ec75b8a71cdde25161c521d1ecaf64b8
-size 1255812224

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/model.safetensors.index.json

@@ -1,772 +0,0 @@
-{
-  "metadata": {
-    "total_size": 16060697664
-  },
-  "weight_map": {
-    "lm_head.weight": "model-00004-of-00004.safetensors",
-    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
-    "model.image_newline": "model-00001-of-00004.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "model.mm_projector.0.bias": "model-00004-of-00004.safetensors",
-    "model.mm_projector.0.weight": "model-00004-of-00004.safetensors",
-    "model.mm_projector.2.bias": "model-00004-of-00004.safetensors",
-    "model.mm_projector.2.weight": "model-00004-of-00004.safetensors",
-    "model.norm.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
-    "model.vision_tower.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors"
-  }
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_0.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:92cc13315f24c28015d695b6cde08bb1cd6fea4cbc435998485ed6fbe4c91285
-size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_1.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f4c154b6a63e0b1f98f7d2847944398f99f1657d35e8eddf7fdf0ae2c24b0552
-size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_2.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f784c6a9507b51189f2caffbd178ea9882103b75852e31c15f47fdae6a43af1d
-size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/rng_state_3.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:34b023e05bc2d12b91dc436d4922b990d50ec8dc56d40dc3e36b3bb34fc81341
-size 15024

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/scheduler.pt

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ce711c5162a77c8e3dc9e306c7c613224d95db6d1eb11552783a6124905667f7
-size 1064

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/special_tokens_map.json

@@ -1,20 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "eos_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/tokenizer_config.json

@@ -1,53 +0,0 @@
-{
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<image>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "right",
-  "processor_class": "LlavaProcessor",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/trainer_state.json

@@ -1,3521 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 0.5714285714285714,
-  "eval_steps": 500,
-  "global_step": 500,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.0,
-      "grad_norm": 8.812040662498683,
-      "learning_rate": 3.7037037037037036e-07,
-      "loss": 0.6324,
-      "step": 1
-    },
-    {
-      "epoch": 0.0,
-      "grad_norm": 8.231021188121307,
-      "learning_rate": 7.407407407407407e-07,
-      "loss": 0.5088,
-      "step": 2
-    },
-    {
-      "epoch": 0.0,
-      "grad_norm": 10.084697585340875,
-      "learning_rate": 1.111111111111111e-06,
-      "loss": 0.5952,
-      "step": 3
-    },
-    {
-      "epoch": 0.0,
-      "grad_norm": 8.504686522060355,
-      "learning_rate": 1.4814814814814815e-06,
-      "loss": 0.5506,
-      "step": 4
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 7.78998673730679,
-      "learning_rate": 1.8518518518518519e-06,
-      "loss": 0.5389,
-      "step": 5
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 7.564344020278276,
-      "learning_rate": 2.222222222222222e-06,
-      "loss": 0.4419,
-      "step": 6
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 6.335657208121732,
-      "learning_rate": 2.5925925925925925e-06,
-      "loss": 0.4503,
-      "step": 7
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 5.736176328651262,
-      "learning_rate": 2.962962962962963e-06,
-      "loss": 0.4152,
-      "step": 8
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 6.646697141146249,
-      "learning_rate": 3.3333333333333333e-06,
-      "loss": 0.4766,
-      "step": 9
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 7.271376765925272,
-      "learning_rate": 3.7037037037037037e-06,
-      "loss": 0.3847,
-      "step": 10
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 11.816959033855433,
-      "learning_rate": 4.074074074074074e-06,
-      "loss": 0.3234,
-      "step": 11
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 17.376716600598993,
-      "learning_rate": 4.444444444444444e-06,
-      "loss": 0.2575,
-      "step": 12
-    },
-    {
-      "epoch": 0.01,
-      "grad_norm": 13.166527782235265,
-      "learning_rate": 4.814814814814815e-06,
-      "loss": 0.3037,
-      "step": 13
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 8.131001484797347,
-      "learning_rate": 5.185185185185185e-06,
-      "loss": 0.1747,
-      "step": 14
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 9.174937982191583,
-      "learning_rate": 5.555555555555557e-06,
-      "loss": 0.2399,
-      "step": 15
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 8.592965454381307,
-      "learning_rate": 5.925925925925926e-06,
-      "loss": 0.1949,
-      "step": 16
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 6.872465363012901,
-      "learning_rate": 6.296296296296297e-06,
-      "loss": 0.1805,
-      "step": 17
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 5.792592320141116,
-      "learning_rate": 6.666666666666667e-06,
-      "loss": 0.1909,
-      "step": 18
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 2.288697729735019,
-      "learning_rate": 7.0370370370370375e-06,
-      "loss": 0.1502,
-      "step": 19
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 1.6947246687987911,
-      "learning_rate": 7.4074074074074075e-06,
-      "loss": 0.1174,
-      "step": 20
-    },
-    {
-      "epoch": 0.02,
-      "grad_norm": 5.72465565098873,
-      "learning_rate": 7.77777777777778e-06,
-      "loss": 0.1739,
-      "step": 21
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 5.765453075575737,
-      "learning_rate": 8.148148148148148e-06,
-      "loss": 0.1894,
-      "step": 22
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 3.2240061796287334,
-      "learning_rate": 8.518518518518519e-06,
-      "loss": 0.1376,
-      "step": 23
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 5.983671039035109,
-      "learning_rate": 8.888888888888888e-06,
-      "loss": 0.1445,
-      "step": 24
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 6.056090509557002,
-      "learning_rate": 9.25925925925926e-06,
-      "loss": 0.1628,
-      "step": 25
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 3.667505927161921,
-      "learning_rate": 9.62962962962963e-06,
-      "loss": 0.1411,
-      "step": 26
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 9.96893801537625,
-      "learning_rate": 1e-05,
-      "loss": 0.1942,
-      "step": 27
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 3.081996759418418,
-      "learning_rate": 9.999965687879942e-06,
-      "loss": 0.1401,
-      "step": 28
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 3.992334648293423,
-      "learning_rate": 9.999862751990697e-06,
-      "loss": 0.1495,
-      "step": 29
-    },
-    {
-      "epoch": 0.03,
-      "grad_norm": 3.446786827543344,
-      "learning_rate": 9.999691193745045e-06,
-      "loss": 0.1356,
-      "step": 30
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 2.452927122228086,
-      "learning_rate": 9.999451015497595e-06,
-      "loss": 0.1576,
-      "step": 31
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 4.8167704565956715,
-      "learning_rate": 9.999142220544759e-06,
-      "loss": 0.1237,
-      "step": 32
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 4.128302969077741,
-      "learning_rate": 9.9987648131247e-06,
-      "loss": 0.124,
-      "step": 33
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 3.7257757203533584,
-      "learning_rate": 9.998318798417276e-06,
-      "loss": 0.1622,
-      "step": 34
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 4.811332863461092,
-      "learning_rate": 9.997804182543973e-06,
-      "loss": 0.1373,
-      "step": 35
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 6.006945777469291,
-      "learning_rate": 9.997220972567815e-06,
-      "loss": 0.1617,
-      "step": 36
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 5.2290233962608275,
-      "learning_rate": 9.996569176493269e-06,
-      "loss": 0.1317,
-      "step": 37
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 4.887606275239368,
-      "learning_rate": 9.995848803266141e-06,
-      "loss": 0.1559,
-      "step": 38
-    },
-    {
-      "epoch": 0.04,
-      "grad_norm": 2.649617040738647,
-      "learning_rate": 9.99505986277344e-06,
-      "loss": 0.1359,
-      "step": 39
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 3.6342992270813492,
-      "learning_rate": 9.994202365843256e-06,
-      "loss": 0.1444,
-      "step": 40
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 2.977502599851725,
-      "learning_rate": 9.993276324244606e-06,
-      "loss": 0.1367,
-      "step": 41
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 3.007957356864356,
-      "learning_rate": 9.992281750687265e-06,
-      "loss": 0.1297,
-      "step": 42
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 1.8406129718892825,
-      "learning_rate": 9.991218658821609e-06,
-      "loss": 0.129,
-      "step": 43
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 2.0044983411925275,
-      "learning_rate": 9.990087063238408e-06,
-      "loss": 0.107,
-      "step": 44
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 1.8795658720176402,
-      "learning_rate": 9.988886979468644e-06,
-      "loss": 0.1121,
-      "step": 45
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 2.356084791458523,
-      "learning_rate": 9.98761842398328e-06,
-      "loss": 0.0981,
-      "step": 46
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 4.534924570867148,
-      "learning_rate": 9.98628141419305e-06,
-      "loss": 0.1794,
-      "step": 47
-    },
-    {
-      "epoch": 0.05,
-      "grad_norm": 2.3579321421491866,
-      "learning_rate": 9.984875968448212e-06,
-      "loss": 0.108,
-      "step": 48
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 2.2510325224717254,
-      "learning_rate": 9.983402106038292e-06,
-      "loss": 0.1117,
-      "step": 49
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 2.4788953336363653,
-      "learning_rate": 9.981859847191827e-06,
-      "loss": 0.1282,
-      "step": 50
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 1.8906655279037572,
-      "learning_rate": 9.980249213076085e-06,
-      "loss": 0.1208,
-      "step": 51
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 3.1073556178878188,
-      "learning_rate": 9.978570225796777e-06,
-      "loss": 0.1644,
-      "step": 52
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 1.9452742167713701,
-      "learning_rate": 9.97682290839775e-06,
-      "loss": 0.1285,
-      "step": 53
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 2.670121793189728,
-      "learning_rate": 9.975007284860664e-06,
-      "loss": 0.142,
-      "step": 54
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 1.8244790066486611,
-      "learning_rate": 9.97312338010468e-06,
-      "loss": 0.1182,
-      "step": 55
-    },
-    {
-      "epoch": 0.06,
-      "grad_norm": 1.1114450183499889,
-      "learning_rate": 9.971171219986104e-06,
-      "loss": 0.1013,
-      "step": 56
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.8735563274335463,
-      "learning_rate": 9.969150831298037e-06,
-      "loss": 0.1351,
-      "step": 57
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.222351118905723,
-      "learning_rate": 9.967062241770008e-06,
-      "loss": 0.1122,
-      "step": 58
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 1.7829363422208375,
-      "learning_rate": 9.964905480067585e-06,
-      "loss": 0.1299,
-      "step": 59
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.187968659410326,
-      "learning_rate": 9.962680575792e-06,
-      "loss": 0.0957,
-      "step": 60
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.846988563675764,
-      "learning_rate": 9.960387559479727e-06,
-      "loss": 0.1187,
-      "step": 61
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 1.643896260993509,
-      "learning_rate": 9.958026462602062e-06,
-      "loss": 0.1086,
-      "step": 62
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.3501454089832894,
-      "learning_rate": 9.955597317564705e-06,
-      "loss": 0.1181,
-      "step": 63
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.3392801570363164,
-      "learning_rate": 9.953100157707299e-06,
-      "loss": 0.1283,
-      "step": 64
-    },
-    {
-      "epoch": 0.07,
-      "grad_norm": 2.215077177644609,
-      "learning_rate": 9.950535017302984e-06,
-      "loss": 0.1194,
-      "step": 65
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.298364445576207,
-      "learning_rate": 9.947901931557924e-06,
-      "loss": 0.1317,
-      "step": 66
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.8138148663788862,
-      "learning_rate": 9.945200936610821e-06,
-      "loss": 0.1241,
-      "step": 67
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.5107542165471934,
-      "learning_rate": 9.942432069532417e-06,
-      "loss": 0.1021,
-      "step": 68
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.5847605764691766,
-      "learning_rate": 9.939595368324996e-06,
-      "loss": 0.1432,
-      "step": 69
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.534356782579585,
-      "learning_rate": 9.936690871921849e-06,
-      "loss": 0.1407,
-      "step": 70
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.2605138628608623,
-      "learning_rate": 9.933718620186745e-06,
-      "loss": 0.1323,
-      "step": 71
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 4.239825098653746,
-      "learning_rate": 9.930678653913392e-06,
-      "loss": 0.1329,
-      "step": 72
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 3.4950058767667134,
-      "learning_rate": 9.927571014824862e-06,
-      "loss": 0.1259,
-      "step": 73
-    },
-    {
-      "epoch": 0.08,
-      "grad_norm": 2.538742560460438,
-      "learning_rate": 9.924395745573029e-06,
-      "loss": 0.1276,
-      "step": 74
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 2.8351129913937987,
-      "learning_rate": 9.921152889737985e-06,
-      "loss": 0.1333,
-      "step": 75
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 3.6729191949488227,
-      "learning_rate": 9.91784249182743e-06,
-      "loss": 0.1209,
-      "step": 76
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 3.5760153753569086,
-      "learning_rate": 9.914464597276072e-06,
-      "loss": 0.1443,
-      "step": 77
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 2.057116589062098,
-      "learning_rate": 9.911019252445e-06,
-      "loss": 0.1082,
-      "step": 78
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 2.6837432634831067,
-      "learning_rate": 9.907506504621052e-06,
-      "loss": 0.1133,
-      "step": 79
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 2.118680599811244,
-      "learning_rate": 9.903926402016153e-06,
-      "loss": 0.1193,
-      "step": 80
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 3.1410450154804663,
-      "learning_rate": 9.90027899376667e-06,
-      "loss": 0.1212,
-      "step": 81
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 5.110344600127584,
-      "learning_rate": 9.896564329932727e-06,
-      "loss": 0.1318,
-      "step": 82
-    },
-    {
-      "epoch": 0.09,
-      "grad_norm": 2.33027322583564,
-      "learning_rate": 9.892782461497521e-06,
-      "loss": 0.1054,
-      "step": 83
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 1.9832010503747017,
-      "learning_rate": 9.88893344036662e-06,
-      "loss": 0.1113,
-      "step": 84
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 2.6705177259808988,
-      "learning_rate": 9.885017319367253e-06,
-      "loss": 0.1181,
-      "step": 85
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 1.6879356709331421,
-      "learning_rate": 9.88103415224759e-06,
-      "loss": 0.0843,
-      "step": 86
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 1.3641501934361187,
-      "learning_rate": 9.87698399367599e-06,
-      "loss": 0.0985,
-      "step": 87
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 2.0451464275509053,
-      "learning_rate": 9.872866899240265e-06,
-      "loss": 0.1057,
-      "step": 88
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 2.2974724008225484,
-      "learning_rate": 9.86868292544691e-06,
-      "loss": 0.1166,
-      "step": 89
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 2.1974043469355458,
-      "learning_rate": 9.864432129720332e-06,
-      "loss": 0.1007,
-      "step": 90
-    },
-    {
-      "epoch": 0.1,
-      "grad_norm": 1.5880779234771585,
-      "learning_rate": 9.860114570402055e-06,
-      "loss": 0.1059,
-      "step": 91
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.6210019246715581,
-      "learning_rate": 9.855730306749924e-06,
-      "loss": 0.0955,
-      "step": 92
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 2.053943766699023,
-      "learning_rate": 9.85127939893729e-06,
-      "loss": 0.1223,
-      "step": 93
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.5230542823178097,
-      "learning_rate": 9.846761908052188e-06,
-      "loss": 0.0969,
-      "step": 94
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.9898055550048976,
-      "learning_rate": 9.842177896096495e-06,
-      "loss": 0.1359,
-      "step": 95
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.5219298537997954,
-      "learning_rate": 9.837527425985074e-06,
-      "loss": 0.1049,
-      "step": 96
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.5694384883915047,
-      "learning_rate": 9.832810561544925e-06,
-      "loss": 0.0885,
-      "step": 97
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 1.8382074625592275,
-      "learning_rate": 9.828027367514296e-06,
-      "loss": 0.0789,
-      "step": 98
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 3.094156783497333,
-      "learning_rate": 9.823177909541795e-06,
-      "loss": 0.125,
-      "step": 99
-    },
-    {
-      "epoch": 0.11,
-      "grad_norm": 2.1177801350294474,
-      "learning_rate": 9.818262254185495e-06,
-      "loss": 0.1286,
-      "step": 100
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 2.051759324600107,
-      "learning_rate": 9.813280468912024e-06,
-      "loss": 0.0867,
-      "step": 101
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 2.309052314502647,
-      "learning_rate": 9.808232622095621e-06,
-      "loss": 0.0836,
-      "step": 102
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 1.9691513403830945,
-      "learning_rate": 9.803118783017221e-06,
-      "loss": 0.1088,
-      "step": 103
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 1.8816504929896276,
-      "learning_rate": 9.797939021863487e-06,
-      "loss": 0.1263,
-      "step": 104
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 3.5814461055271196,
-      "learning_rate": 9.792693409725853e-06,
-      "loss": 0.1266,
-      "step": 105
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 2.804380821700463,
-      "learning_rate": 9.78738201859955e-06,
-      "loss": 0.1548,
-      "step": 106
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 2.0086077809415332,
-      "learning_rate": 9.782004921382612e-06,
-      "loss": 0.1312,
-      "step": 107
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 1.7232244244365362,
-      "learning_rate": 9.776562191874884e-06,
-      "loss": 0.1197,
-      "step": 108
-    },
-    {
-      "epoch": 0.12,
-      "grad_norm": 1.3610013356293815,
-      "learning_rate": 9.771053904776998e-06,
-      "loss": 0.1139,
-      "step": 109
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 3.0085151960644643,
-      "learning_rate": 9.765480135689357e-06,
-      "loss": 0.1049,
-      "step": 110
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 2.94972805872663,
-      "learning_rate": 9.759840961111098e-06,
-      "loss": 0.0992,
-      "step": 111
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 1.2797796597040898,
-      "learning_rate": 9.754136458439033e-06,
-      "loss": 0.0779,
-      "step": 112
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 2.080098377998089,
-      "learning_rate": 9.748366705966595e-06,
-      "loss": 0.1192,
-      "step": 113
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 3.026465124623598,
-      "learning_rate": 9.742531782882758e-06,
-      "loss": 0.1248,
-      "step": 114
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 3.419083049509824,
-      "learning_rate": 9.736631769270958e-06,
-      "loss": 0.1284,
-      "step": 115
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 1.9998224613269435,
-      "learning_rate": 9.730666746107982e-06,
-      "loss": 0.1106,
-      "step": 116
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 1.7572924951035829,
-      "learning_rate": 9.724636795262868e-06,
-      "loss": 0.0874,
-      "step": 117
-    },
-    {
-      "epoch": 0.13,
-      "grad_norm": 2.448977492181066,
-      "learning_rate": 9.718541999495773e-06,
-      "loss": 0.0823,
-      "step": 118
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 4.110748776288409,
-      "learning_rate": 9.712382442456845e-06,
-      "loss": 0.1604,
-      "step": 119
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 2.09981079064145,
-      "learning_rate": 9.706158208685066e-06,
-      "loss": 0.0911,
-      "step": 120
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 1.9370008400745906,
-      "learning_rate": 9.6998693836071e-06,
-      "loss": 0.1122,
-      "step": 121
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 1.9316495646897083,
-      "learning_rate": 9.693516053536118e-06,
-      "loss": 0.0807,
-      "step": 122
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 3.16334665341287,
-      "learning_rate": 9.687098305670606e-06,
-      "loss": 0.1276,
-      "step": 123
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 2.341234757750837,
-      "learning_rate": 9.680616228093178e-06,
-      "loss": 0.1187,
-      "step": 124
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 2.223298454459867,
-      "learning_rate": 9.674069909769365e-06,
-      "loss": 0.1045,
-      "step": 125
-    },
-    {
-      "epoch": 0.14,
-      "grad_norm": 3.4945449518796585,
-      "learning_rate": 9.66745944054639e-06,
-      "loss": 0.12,
-      "step": 126
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 4.143984048508036,
-      "learning_rate": 9.66078491115194e-06,
-      "loss": 0.1013,
-      "step": 127
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 2.7705806851123493,
-      "learning_rate": 9.654046413192916e-06,
-      "loss": 0.1123,
-      "step": 128
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 1.2313407498327578,
-      "learning_rate": 9.647244039154178e-06,
-      "loss": 0.0846,
-      "step": 129
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 1.7722835766792302,
-      "learning_rate": 9.640377882397276e-06,
-      "loss": 0.0965,
-      "step": 130
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 3.4978175367401065,
-      "learning_rate": 9.633448037159167e-06,
-      "loss": 0.1218,
-      "step": 131
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 3.9772864819418463,
-      "learning_rate": 9.626454598550927e-06,
-      "loss": 0.0919,
-      "step": 132
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 1.9729154551618289,
-      "learning_rate": 9.619397662556434e-06,
-      "loss": 0.1104,
-      "step": 133
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 1.9749670072044205,
-      "learning_rate": 9.612277326031065e-06,
-      "loss": 0.0879,
-      "step": 134
-    },
-    {
-      "epoch": 0.15,
-      "grad_norm": 2.46650550465165,
-      "learning_rate": 9.605093686700356e-06,
-      "loss": 0.1151,
-      "step": 135
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 2.1700672021295397,
-      "learning_rate": 9.597846843158663e-06,
-      "loss": 0.1192,
-      "step": 136
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 1.5484117589367528,
-      "learning_rate": 9.590536894867814e-06,
-      "loss": 0.1093,
-      "step": 137
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 3.159315025435744,
-      "learning_rate": 9.58316394215574e-06,
-      "loss": 0.1246,
-      "step": 138
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 2.3062290760264044,
-      "learning_rate": 9.575728086215093e-06,
-      "loss": 0.1044,
-      "step": 139
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 2.1924424597130647,
-      "learning_rate": 9.568229429101867e-06,
-      "loss": 0.1018,
-      "step": 140
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 1.4292889067167893,
-      "learning_rate": 9.560668073733993e-06,
-      "loss": 0.1102,
-      "step": 141
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 1.8452890182403634,
-      "learning_rate": 9.553044123889922e-06,
-      "loss": 0.0851,
-      "step": 142
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 1.686312012130693,
-      "learning_rate": 9.54535768420721e-06,
-      "loss": 0.0877,
-      "step": 143
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 1.9544040435460412,
-      "learning_rate": 9.537608860181069e-06,
-      "loss": 0.1401,
-      "step": 144
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.5638709576265777,
-      "learning_rate": 9.529797758162935e-06,
-      "loss": 0.1092,
-      "step": 145
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 3.282994763913454,
-      "learning_rate": 9.521924485358993e-06,
-      "loss": 0.1153,
-      "step": 146
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.5304046735735803,
-      "learning_rate": 9.513989149828718e-06,
-      "loss": 0.1249,
-      "step": 147
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.084105660790946,
-      "learning_rate": 9.505991860483384e-06,
-      "loss": 0.1127,
-      "step": 148
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.0807266175742334,
-      "learning_rate": 9.497932727084571e-06,
-      "loss": 0.0923,
-      "step": 149
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.535491149619429,
-      "learning_rate": 9.489811860242658e-06,
-      "loss": 0.1171,
-      "step": 150
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.8577061917206694,
-      "learning_rate": 9.481629371415315e-06,
-      "loss": 0.1027,
-      "step": 151
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 2.366848987135643,
-      "learning_rate": 9.47338537290595e-06,
-      "loss": 0.1461,
-      "step": 152
-    },
-    {
-      "epoch": 0.17,
-      "grad_norm": 1.370601085821976,
-      "learning_rate": 9.465079977862193e-06,
-      "loss": 0.1176,
-      "step": 153
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 2.418308466667954,
-      "learning_rate": 9.45671330027433e-06,
-      "loss": 0.14,
-      "step": 154
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 2.3789531359503155,
-      "learning_rate": 9.448285454973739e-06,
-      "loss": 0.1272,
-      "step": 155
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 1.837535532086006,
-      "learning_rate": 9.439796557631311e-06,
-      "loss": 0.1018,
-      "step": 156
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 2.26111781111186,
-      "learning_rate": 9.431246724755879e-06,
-      "loss": 0.1089,
-      "step": 157
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 2.149991918092674,
-      "learning_rate": 9.422636073692595e-06,
-      "loss": 0.1224,
-      "step": 158
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 1.7257130483628491,
-      "learning_rate": 9.413964722621339e-06,
-      "loss": 0.0829,
-      "step": 159
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 1.8121675919179852,
-      "learning_rate": 9.405232790555083e-06,
-      "loss": 0.1082,
-      "step": 160
-    },
-    {
-      "epoch": 0.18,
-      "grad_norm": 1.1269467977377563,
-      "learning_rate": 9.396440397338273e-06,
-      "loss": 0.0735,
-      "step": 161
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 2.66841905851172,
-      "learning_rate": 9.387587663645163e-06,
-      "loss": 0.1171,
-      "step": 162
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 1.9979957509268915,
-      "learning_rate": 9.378674710978185e-06,
-      "loss": 0.0867,
-      "step": 163
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 1.3467231541181564,
-      "learning_rate": 9.369701661666255e-06,
-      "loss": 0.084,
-      "step": 164
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 1.5661741867693126,
-      "learning_rate": 9.36066863886311e-06,
-      "loss": 0.0879,
-      "step": 165
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 1.4781065347920526,
-      "learning_rate": 9.35157576654562e-06,
-      "loss": 0.081,
-      "step": 166
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 1.747486649844575,
-      "learning_rate": 9.342423169512072e-06,
-      "loss": 0.0998,
-      "step": 167
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 2.183425315531194,
-      "learning_rate": 9.333210973380469e-06,
-      "loss": 0.113,
-      "step": 168
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 2.6187200801222126,
-      "learning_rate": 9.323939304586806e-06,
-      "loss": 0.1211,
-      "step": 169
-    },
-    {
-      "epoch": 0.19,
-      "grad_norm": 2.298579486613351,
-      "learning_rate": 9.314608290383324e-06,
-      "loss": 0.098,
-      "step": 170
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 2.3760059355851286,
-      "learning_rate": 9.305218058836778e-06,
-      "loss": 0.1072,
-      "step": 171
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 2.7212461615360892,
-      "learning_rate": 9.295768738826668e-06,
-      "loss": 0.1128,
-      "step": 172
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 1.7379020422811915,
-      "learning_rate": 9.286260460043475e-06,
-      "loss": 0.125,
-      "step": 173
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 3.1792312593675724,
-      "learning_rate": 9.27669335298688e-06,
-      "loss": 0.124,
-      "step": 174
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 2.0277111530618916,
-      "learning_rate": 9.267067548963975e-06,
-      "loss": 0.0942,
-      "step": 175
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 1.4469039936621195,
-      "learning_rate": 9.257383180087454e-06,
-      "loss": 0.0909,
-      "step": 176
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 2.051541742743329,
-      "learning_rate": 9.24764037927381e-06,
-      "loss": 0.1199,
-      "step": 177
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 1.4918383860547757,
-      "learning_rate": 9.237839280241504e-06,
-      "loss": 0.0948,
-      "step": 178
-    },
-    {
-      "epoch": 0.2,
-      "grad_norm": 1.64559721928272,
-      "learning_rate": 9.22798001750913e-06,
-      "loss": 0.0926,
-      "step": 179
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 2.7118844975893306,
-      "learning_rate": 9.218062726393572e-06,
-      "loss": 0.1006,
-      "step": 180
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 1.9102131867075969,
-      "learning_rate": 9.208087543008142e-06,
-      "loss": 0.0971,
-      "step": 181
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 1.1482279190562423,
-      "learning_rate": 9.198054604260714e-06,
-      "loss": 0.0856,
-      "step": 182
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 0.8541863535744467,
-      "learning_rate": 9.187964047851851e-06,
-      "loss": 0.0744,
-      "step": 183
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 1.6754271851342437,
-      "learning_rate": 9.177816012272904e-06,
-      "loss": 0.1079,
-      "step": 184
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 7.473478229947605,
-      "learning_rate": 9.16761063680412e-06,
-      "loss": 0.0957,
-      "step": 185
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 3.0632873310708706,
-      "learning_rate": 9.157348061512728e-06,
-      "loss": 0.1259,
-      "step": 186
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 2.173245370297517,
-      "learning_rate": 9.14702842725101e-06,
-      "loss": 0.1064,
-      "step": 187
-    },
-    {
-      "epoch": 0.21,
-      "grad_norm": 2.5577392058893906,
-      "learning_rate": 9.136651875654381e-06,
-      "loss": 0.1018,
-      "step": 188
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 2.4914753007974073,
-      "learning_rate": 9.126218549139434e-06,
-      "loss": 0.0766,
-      "step": 189
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 2.607548007532628,
-      "learning_rate": 9.115728590901988e-06,
-      "loss": 0.1145,
-      "step": 190
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 2.3652955179863704,
-      "learning_rate": 9.10518214491513e-06,
-      "loss": 0.1075,
-      "step": 191
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 1.944852075113964,
-      "learning_rate": 9.094579355927224e-06,
-      "loss": 0.1237,
-      "step": 192
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 2.531422123532763,
-      "learning_rate": 9.083920369459941e-06,
-      "loss": 0.0996,
-      "step": 193
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 1.6840091229079706,
-      "learning_rate": 9.073205331806248e-06,
-      "loss": 0.0978,
-      "step": 194
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 1.1180165313908608,
-      "learning_rate": 9.062434390028407e-06,
-      "loss": 0.1071,
-      "step": 195
-    },
-    {
-      "epoch": 0.22,
-      "grad_norm": 1.3064591316530667,
-      "learning_rate": 9.05160769195596e-06,
-      "loss": 0.0989,
-      "step": 196
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 2.467504354977469,
-      "learning_rate": 9.040725386183691e-06,
-      "loss": 0.1042,
-      "step": 197
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 2.050946499964558,
-      "learning_rate": 9.029787622069594e-06,
-      "loss": 0.0975,
-      "step": 198
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.4499865119155375,
-      "learning_rate": 9.018794549732819e-06,
-      "loss": 0.085,
-      "step": 199
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.3533643803726048,
-      "learning_rate": 9.007746320051612e-06,
-      "loss": 0.1035,
-      "step": 200
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.6026335880678133,
-      "learning_rate": 8.996643084661245e-06,
-      "loss": 0.1027,
-      "step": 201
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.9397686652721016,
-      "learning_rate": 8.985484995951941e-06,
-      "loss": 0.1186,
-      "step": 202
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.985748151579965,
-      "learning_rate": 8.974272207066767e-06,
-      "loss": 0.1106,
-      "step": 203
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 3.0494865642540487,
-      "learning_rate": 8.96300487189955e-06,
-      "loss": 0.1064,
-      "step": 204
-    },
-    {
-      "epoch": 0.23,
-      "grad_norm": 1.5961078519634195,
-      "learning_rate": 8.951683145092749e-06,
-      "loss": 0.107,
-      "step": 205
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 1.1110307819462288,
-      "learning_rate": 8.940307182035348e-06,
-      "loss": 0.0572,
-      "step": 206
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 1.5666032654284987,
-      "learning_rate": 8.928877138860708e-06,
-      "loss": 0.0855,
-      "step": 207
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 2.5628141099754145,
-      "learning_rate": 8.917393172444435e-06,
-      "loss": 0.1166,
-      "step": 208
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 2.1688262602344186,
-      "learning_rate": 8.905855440402225e-06,
-      "loss": 0.1282,
-      "step": 209
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 1.9861109578632914,
-      "learning_rate": 8.894264101087692e-06,
-      "loss": 0.114,
-      "step": 210
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 2.501921488076984,
-      "learning_rate": 8.882619313590212e-06,
-      "loss": 0.1097,
-      "step": 211
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 2.268674705211975,
-      "learning_rate": 8.870921237732722e-06,
-      "loss": 0.1071,
-      "step": 212
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 2.067986488761189,
-      "learning_rate": 8.859170034069533e-06,
-      "loss": 0.078,
-      "step": 213
-    },
-    {
-      "epoch": 0.24,
-      "grad_norm": 1.848889166285347,
-      "learning_rate": 8.847365863884131e-06,
-      "loss": 0.0939,
-      "step": 214
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 1.633820214871211,
-      "learning_rate": 8.835508889186957e-06,
-      "loss": 0.0668,
-      "step": 215
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 1.762882965258614,
-      "learning_rate": 8.823599272713188e-06,
-      "loss": 0.1218,
-      "step": 216
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 1.320523619911371,
-      "learning_rate": 8.8116371779205e-06,
-      "loss": 0.0862,
-      "step": 217
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 2.2237720260978944,
-      "learning_rate": 8.799622768986826e-06,
-      "loss": 0.0818,
-      "step": 218
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 2.4364710210798948,
-      "learning_rate": 8.787556210808101e-06,
-      "loss": 0.1199,
-      "step": 219
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 1.6849135689019348,
-      "learning_rate": 8.775437668996004e-06,
-      "loss": 0.1075,
-      "step": 220
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 2.220372344984879,
-      "learning_rate": 8.76326730987568e-06,
-      "loss": 0.0939,
-      "step": 221
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 1.5555946268423595,
-      "learning_rate": 8.751045300483457e-06,
-      "loss": 0.0936,
-      "step": 222
-    },
-    {
-      "epoch": 0.25,
-      "grad_norm": 2.1402335410022744,
-      "learning_rate": 8.738771808564555e-06,
-      "loss": 0.0934,
-      "step": 223
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.0488112883529606,
-      "learning_rate": 8.726447002570791e-06,
-      "loss": 0.0984,
-      "step": 224
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.6002022312116384,
-      "learning_rate": 8.714071051658247e-06,
-      "loss": 0.1182,
-      "step": 225
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.1565321002590416,
-      "learning_rate": 8.701644125684972e-06,
-      "loss": 0.0936,
-      "step": 226
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 2.185324721324382,
-      "learning_rate": 8.689166395208638e-06,
-      "loss": 0.083,
-      "step": 227
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 2.030700715820133,
-      "learning_rate": 8.676638031484196e-06,
-      "loss": 0.117,
-      "step": 228
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.984880186535839,
-      "learning_rate": 8.664059206461537e-06,
-      "loss": 0.1018,
-      "step": 229
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.5970035045867854,
-      "learning_rate": 8.65143009278312e-06,
-      "loss": 0.0949,
-      "step": 230
-    },
-    {
-      "epoch": 0.26,
-      "grad_norm": 1.1816220314739803,
-      "learning_rate": 8.638750863781614e-06,
-      "loss": 0.0813,
-      "step": 231
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.1265530666409853,
-      "learning_rate": 8.626021693477506e-06,
-      "loss": 0.0807,
-      "step": 232
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.4798060764319763,
-      "learning_rate": 8.613242756576729e-06,
-      "loss": 0.1005,
-      "step": 233
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 2.2654194933943224,
-      "learning_rate": 8.600414228468245e-06,
-      "loss": 0.1188,
-      "step": 234
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 2.069029364915602,
-      "learning_rate": 8.587536285221656e-06,
-      "loss": 0.1174,
-      "step": 235
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.575346902419333,
-      "learning_rate": 8.574609103584776e-06,
-      "loss": 0.1085,
-      "step": 236
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.4098282704868057,
-      "learning_rate": 8.561632860981205e-06,
-      "loss": 0.0866,
-      "step": 237
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.322192828067451,
-      "learning_rate": 8.548607735507903e-06,
-      "loss": 0.0989,
-      "step": 238
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 2.503973593981987,
-      "learning_rate": 8.535533905932739e-06,
-      "loss": 0.0959,
-      "step": 239
-    },
-    {
-      "epoch": 0.27,
-      "grad_norm": 1.9979798111827445,
-      "learning_rate": 8.522411551692034e-06,
-      "loss": 0.1173,
-      "step": 240
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 2.331120139565699,
-      "learning_rate": 8.509240852888106e-06,
-      "loss": 0.1207,
-      "step": 241
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 2.6853298265250545,
-      "learning_rate": 8.496021990286799e-06,
-      "loss": 0.1062,
-      "step": 242
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.9768348877886996,
-      "learning_rate": 8.482755145314987e-06,
-      "loss": 0.1072,
-      "step": 243
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 2.6831710659010493,
-      "learning_rate": 8.469440500058104e-06,
-      "loss": 0.1415,
-      "step": 244
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.3075146767802872,
-      "learning_rate": 8.456078237257632e-06,
-      "loss": 0.0927,
-      "step": 245
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.6603378900652728,
-      "learning_rate": 8.442668540308597e-06,
-      "loss": 0.1199,
-      "step": 246
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.732311993009954,
-      "learning_rate": 8.429211593257054e-06,
-      "loss": 0.1162,
-      "step": 247
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.0678226132557302,
-      "learning_rate": 8.415707580797552e-06,
-      "loss": 0.0811,
-      "step": 248
-    },
-    {
-      "epoch": 0.28,
-      "grad_norm": 1.637868487774632,
-      "learning_rate": 8.402156688270613e-06,
-      "loss": 0.1134,
-      "step": 249
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 2.133244302298436,
-      "learning_rate": 8.388559101660177e-06,
-      "loss": 0.0926,
-      "step": 250
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 2.0183612078901554,
-      "learning_rate": 8.374915007591053e-06,
-      "loss": 0.1111,
-      "step": 251
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 1.9309545129494814,
-      "learning_rate": 8.361224593326359e-06,
-      "loss": 0.0935,
-      "step": 252
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 1.9872264464717466,
-      "learning_rate": 8.347488046764948e-06,
-      "loss": 0.0943,
-      "step": 253
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 3.421297893280941,
-      "learning_rate": 8.33370555643884e-06,
-      "loss": 0.1065,
-      "step": 254
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 2.897377603677577,
-      "learning_rate": 8.319877311510614e-06,
-      "loss": 0.1002,
-      "step": 255
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 2.3046015963660937,
-      "learning_rate": 8.30600350177083e-06,
-      "loss": 0.0855,
-      "step": 256
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 2.124569836934495,
-      "learning_rate": 8.29208431763542e-06,
-      "loss": 0.1393,
-      "step": 257
-    },
-    {
-      "epoch": 0.29,
-      "grad_norm": 1.1578644869843313,
-      "learning_rate": 8.278119950143074e-06,
-      "loss": 0.0727,
-      "step": 258
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 1.854683458939737,
-      "learning_rate": 8.264110590952609e-06,
-      "loss": 0.0943,
-      "step": 259
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 1.4832089308281162,
-      "learning_rate": 8.250056432340352e-06,
-      "loss": 0.0874,
-      "step": 260
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 1.258489452559218,
-      "learning_rate": 8.235957667197494e-06,
-      "loss": 0.0813,
-      "step": 261
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 1.8439098458745395,
-      "learning_rate": 8.221814489027447e-06,
-      "loss": 0.1109,
-      "step": 262
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 2.044799826590439,
-      "learning_rate": 8.207627091943178e-06,
-      "loss": 0.1031,
-      "step": 263
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 1.0827682370362863,
-      "learning_rate": 8.193395670664555e-06,
-      "loss": 0.1009,
-      "step": 264
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 2.323956787231295,
-      "learning_rate": 8.179120420515675e-06,
-      "loss": 0.124,
-      "step": 265
-    },
-    {
-      "epoch": 0.3,
-      "grad_norm": 10.023030336667428,
-      "learning_rate": 8.164801537422179e-06,
-      "loss": 0.0942,
-      "step": 266
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.7106775924129292,
-      "learning_rate": 8.150439217908557e-06,
-      "loss": 0.1294,
-      "step": 267
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 2.496815910955381,
-      "learning_rate": 8.136033659095462e-06,
-      "loss": 0.1046,
-      "step": 268
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.646786991911152,
-      "learning_rate": 8.121585058697e-06,
-      "loss": 0.0861,
-      "step": 269
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.8266200161202253,
-      "learning_rate": 8.107093615018017e-06,
-      "loss": 0.0949,
-      "step": 270
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 2.5916169021461823,
-      "learning_rate": 8.092559526951374e-06,
-      "loss": 0.1169,
-      "step": 271
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.3232237535990277,
-      "learning_rate": 8.077982993975222e-06,
-      "loss": 0.0972,
-      "step": 272
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.5105056997202508,
-      "learning_rate": 8.063364216150258e-06,
-      "loss": 0.0838,
-      "step": 273
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.422000688554186,
-      "learning_rate": 8.04870339411699e-06,
-      "loss": 0.0906,
-      "step": 274
-    },
-    {
-      "epoch": 0.31,
-      "grad_norm": 1.6597229035037346,
-      "learning_rate": 8.034000729092967e-06,
-      "loss": 0.1005,
-      "step": 275
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 2.0117580401941204,
-      "learning_rate": 8.019256422870036e-06,
-      "loss": 0.1114,
-      "step": 276
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 1.1092429217293147,
-      "learning_rate": 8.00447067781156e-06,
-      "loss": 0.0879,
-      "step": 277
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 0.7979548894678264,
-      "learning_rate": 7.98964369684964e-06,
-      "loss": 0.0802,
-      "step": 278
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 4.002883666778827,
-      "learning_rate": 7.974775683482337e-06,
-      "loss": 0.1201,
-      "step": 279
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 2.9158794261385665,
-      "learning_rate": 7.959866841770875e-06,
-      "loss": 0.0928,
-      "step": 280
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 1.8564270859008971,
-      "learning_rate": 7.94491737633684e-06,
-      "loss": 0.1002,
-      "step": 281
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 0.9683845320254458,
-      "learning_rate": 7.929927492359377e-06,
-      "loss": 0.076,
-      "step": 282
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 1.4081959120743777,
-      "learning_rate": 7.914897395572362e-06,
-      "loss": 0.1062,
-      "step": 283
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 2.5215551080759475,
-      "learning_rate": 7.899827292261589e-06,
-      "loss": 0.1165,
-      "step": 284
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.2748703118190907,
-      "learning_rate": 7.884717389261935e-06,
-      "loss": 0.0815,
-      "step": 285
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.7476333672006885,
-      "learning_rate": 7.869567893954523e-06,
-      "loss": 0.1036,
-      "step": 286
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 2.0867624034561008,
-      "learning_rate": 7.854379014263877e-06,
-      "loss": 0.0975,
-      "step": 287
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.505523226821333,
-      "learning_rate": 7.839150958655058e-06,
-      "loss": 0.0909,
-      "step": 288
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.1729694098449017,
-      "learning_rate": 7.823883936130817e-06,
-      "loss": 0.0973,
-      "step": 289
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 2.8384149448507516,
-      "learning_rate": 7.808578156228718e-06,
-      "loss": 0.0897,
-      "step": 290
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.62588050433828,
-      "learning_rate": 7.793233829018263e-06,
-      "loss": 0.0743,
-      "step": 291
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 1.3669908142605618,
-      "learning_rate": 7.777851165098012e-06,
-      "loss": 0.0905,
-      "step": 292
-    },
-    {
-      "epoch": 0.33,
-      "grad_norm": 2.808747017133476,
-      "learning_rate": 7.762430375592689e-06,
-      "loss": 0.1093,
-      "step": 293
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 1.6645286286403973,
-      "learning_rate": 7.746971672150286e-06,
-      "loss": 0.0904,
-      "step": 294
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 1.5994325863369407,
-      "learning_rate": 7.731475266939159e-06,
-      "loss": 0.1028,
-      "step": 295
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 1.5901303888596037,
-      "learning_rate": 7.715941372645115e-06,
-      "loss": 0.0956,
-      "step": 296
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 2.328737208260307,
-      "learning_rate": 7.70037020246849e-06,
-      "loss": 0.1105,
-      "step": 297
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 1.9563743002247425,
-      "learning_rate": 7.68476197012123e-06,
-      "loss": 0.1004,
-      "step": 298
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 2.0781455779725273,
-      "learning_rate": 7.669116889823955e-06,
-      "loss": 0.0996,
-      "step": 299
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 0.9436403770848232,
-      "learning_rate": 7.653435176303007e-06,
-      "loss": 0.0761,
-      "step": 300
-    },
-    {
-      "epoch": 0.34,
-      "grad_norm": 1.6406510415551603,
-      "learning_rate": 7.637717044787527e-06,
-      "loss": 0.1147,
-      "step": 301
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.7565088827574487,
-      "learning_rate": 7.621962711006479e-06,
-      "loss": 0.1064,
-      "step": 302
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.744343584962407,
-      "learning_rate": 7.6061723911857e-06,
-      "loss": 0.1237,
-      "step": 303
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.8449164911800016,
-      "learning_rate": 7.59034630204493e-06,
-      "loss": 0.0866,
-      "step": 304
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.950623541879017,
-      "learning_rate": 7.574484660794836e-06,
-      "loss": 0.0933,
-      "step": 305
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 2.1686948556536465,
-      "learning_rate": 7.558587685134035e-06,
-      "loss": 0.1025,
-      "step": 306
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.7581227603244038,
-      "learning_rate": 7.542655593246103e-06,
-      "loss": 0.0875,
-      "step": 307
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 2.067970224647301,
-      "learning_rate": 7.526688603796578e-06,
-      "loss": 0.0984,
-      "step": 308
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 1.589283294901552,
-      "learning_rate": 7.510686935929963e-06,
-      "loss": 0.086,
-      "step": 309
-    },
-    {
-      "epoch": 0.35,
-      "grad_norm": 2.0302397726328767,
-      "learning_rate": 7.494650809266719e-06,
-      "loss": 0.099,
-      "step": 310
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.5989723290365598,
-      "learning_rate": 7.478580443900247e-06,
-      "loss": 0.0939,
-      "step": 311
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.7617760156145226,
-      "learning_rate": 7.4624760603938704e-06,
-      "loss": 0.1073,
-      "step": 312
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 2.399035341306831,
-      "learning_rate": 7.446337879777803e-06,
-      "loss": 0.0875,
-      "step": 313
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.7044340741166755,
-      "learning_rate": 7.430166123546122e-06,
-      "loss": 0.1059,
-      "step": 314
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 2.0742510793830444,
-      "learning_rate": 7.413961013653725e-06,
-      "loss": 0.121,
-      "step": 315
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.3710310648899238,
-      "learning_rate": 7.397722772513282e-06,
-      "loss": 0.099,
-      "step": 316
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.4093925149964166,
-      "learning_rate": 7.381451622992184e-06,
-      "loss": 0.0809,
-      "step": 317
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 1.0871938974349964,
-      "learning_rate": 7.3651477884094855e-06,
-      "loss": 0.0873,
-      "step": 318
-    },
-    {
-      "epoch": 0.36,
-      "grad_norm": 2.6023982808593678,
-      "learning_rate": 7.34881149253284e-06,
-      "loss": 0.0827,
-      "step": 319
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 2.467883768091289,
-      "learning_rate": 7.332442959575424e-06,
-      "loss": 0.1005,
-      "step": 320
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.1212388969162672,
-      "learning_rate": 7.316042414192866e-06,
-      "loss": 0.0914,
-      "step": 321
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.8358020056483744,
-      "learning_rate": 7.299610081480155e-06,
-      "loss": 0.1146,
-      "step": 322
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.3636541396641189,
-      "learning_rate": 7.283146186968566e-06,
-      "loss": 0.1085,
-      "step": 323
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.07569146070376,
-      "learning_rate": 7.266650956622546e-06,
-      "loss": 0.089,
-      "step": 324
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.8768958532173836,
-      "learning_rate": 7.2501246168366235e-06,
-      "loss": 0.096,
-      "step": 325
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.8347653173611815,
-      "learning_rate": 7.233567394432302e-06,
-      "loss": 0.1067,
-      "step": 326
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.202389845113578,
-      "learning_rate": 7.216979516654944e-06,
-      "loss": 0.0754,
-      "step": 327
-    },
-    {
-      "epoch": 0.37,
-      "grad_norm": 1.0247024117157115,
-      "learning_rate": 7.200361211170648e-06,
-      "loss": 0.0958,
-      "step": 328
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 0.9758438803344642,
-      "learning_rate": 7.183712706063133e-06,
-      "loss": 0.073,
-      "step": 329
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.0927982452541785,
-      "learning_rate": 7.1670342298306e-06,
-      "loss": 0.0891,
-      "step": 330
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.3702951110134494,
-      "learning_rate": 7.1503260113826035e-06,
-      "loss": 0.102,
-      "step": 331
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.6194293721517825,
-      "learning_rate": 7.1335882800369e-06,
-      "loss": 0.0877,
-      "step": 332
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.3330024442939468,
-      "learning_rate": 7.116821265516307e-06,
-      "loss": 0.1038,
-      "step": 333
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.5082089283588793,
-      "learning_rate": 7.100025197945555e-06,
-      "loss": 0.0903,
-      "step": 334
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 0.918479171265335,
-      "learning_rate": 7.083200307848116e-06,
-      "loss": 0.0782,
-      "step": 335
-    },
-    {
-      "epoch": 0.38,
-      "grad_norm": 1.270870459177953,
-      "learning_rate": 7.066346826143049e-06,
-      "loss": 0.078,
-      "step": 336
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 2.808167569407084,
-      "learning_rate": 7.049464984141831e-06,
-      "loss": 0.1034,
-      "step": 337
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.4850123445239072,
-      "learning_rate": 7.032555013545176e-06,
-      "loss": 0.0987,
-      "step": 338
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.0857390354428968,
-      "learning_rate": 7.015617146439863e-06,
-      "loss": 0.0846,
-      "step": 339
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.4019142617304745,
-      "learning_rate": 6.998651615295541e-06,
-      "loss": 0.0946,
-      "step": 340
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.4419350588759805,
-      "learning_rate": 6.9816586529615475e-06,
-      "loss": 0.0925,
-      "step": 341
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 2.122469313937723,
-      "learning_rate": 6.9646384926637076e-06,
-      "loss": 0.101,
-      "step": 342
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 2.023493438132179,
-      "learning_rate": 6.947591368001138e-06,
-      "loss": 0.1049,
-      "step": 343
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.8085392355883325,
-      "learning_rate": 6.930517512943029e-06,
-      "loss": 0.0906,
-      "step": 344
-    },
-    {
-      "epoch": 0.39,
-      "grad_norm": 1.8476500146983619,
-      "learning_rate": 6.913417161825449e-06,
-      "loss": 0.075,
-      "step": 345
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.3169871494739973,
-      "learning_rate": 6.896290549348119e-06,
-      "loss": 0.0999,
-      "step": 346
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.5793887545549115,
-      "learning_rate": 6.879137910571191e-06,
-      "loss": 0.0951,
-      "step": 347
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.8170746345841167,
-      "learning_rate": 6.861959480912027e-06,
-      "loss": 0.0872,
-      "step": 348
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.9489463937624985,
-      "learning_rate": 6.8447554961419615e-06,
-      "loss": 0.1034,
-      "step": 349
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 2.1832874089959704,
-      "learning_rate": 6.82752619238307e-06,
-      "loss": 0.1158,
-      "step": 350
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.3585943935815048,
-      "learning_rate": 6.810271806104931e-06,
-      "loss": 0.0801,
-      "step": 351
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 4.249666680648082,
-      "learning_rate": 6.792992574121374e-06,
-      "loss": 0.097,
-      "step": 352
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 2.597130012884428,
-      "learning_rate": 6.7756887335872275e-06,
-      "loss": 0.1166,
-      "step": 353
-    },
-    {
-      "epoch": 0.4,
-      "grad_norm": 1.4936397575141562,
-      "learning_rate": 6.758360521995079e-06,
-      "loss": 0.1212,
-      "step": 354
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.171091229180981,
-      "learning_rate": 6.741008177171995e-06,
-      "loss": 0.0908,
-      "step": 355
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 0.9253783941058572,
-      "learning_rate": 6.723631937276271e-06,
-      "loss": 0.0715,
-      "step": 356
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.1914008404922645,
-      "learning_rate": 6.706232040794162e-06,
-      "loss": 0.0753,
-      "step": 357
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.6494093340509537,
-      "learning_rate": 6.6888087265365995e-06,
-      "loss": 0.1066,
-      "step": 358
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.1812819049319463,
-      "learning_rate": 6.671362233635926e-06,
-      "loss": 0.0927,
-      "step": 359
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 2.4559536958240527,
-      "learning_rate": 6.653892801542604e-06,
-      "loss": 0.0928,
-      "step": 360
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.25389273102752,
-      "learning_rate": 6.636400670021934e-06,
-      "loss": 0.0999,
-      "step": 361
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.7179424428138383,
-      "learning_rate": 6.618886079150764e-06,
-      "loss": 0.0921,
-      "step": 362
-    },
-    {
-      "epoch": 0.41,
-      "grad_norm": 1.135878355877357,
-      "learning_rate": 6.601349269314188e-06,
-      "loss": 0.082,
-      "step": 363
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.362482025104223,
-      "learning_rate": 6.583790481202261e-06,
-      "loss": 0.0999,
-      "step": 364
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.9432112152221168,
-      "learning_rate": 6.566209955806679e-06,
-      "loss": 0.1061,
-      "step": 365
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.6175027458716444,
-      "learning_rate": 6.54860793441748e-06,
-      "loss": 0.1199,
-      "step": 366
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.7771666465783587,
-      "learning_rate": 6.530984658619735e-06,
-      "loss": 0.1045,
-      "step": 367
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.4987987433398124,
-      "learning_rate": 6.5133403702902245e-06,
-      "loss": 0.1011,
-      "step": 368
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 2.024080574349673,
-      "learning_rate": 6.495675311594123e-06,
-      "loss": 0.0962,
-      "step": 369
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.233698410707572,
-      "learning_rate": 6.477989724981679e-06,
-      "loss": 0.102,
-      "step": 370
-    },
-    {
-      "epoch": 0.42,
-      "grad_norm": 1.860410897978535,
-      "learning_rate": 6.46028385318488e-06,
-      "loss": 0.0882,
-      "step": 371
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.056573638183693,
-      "learning_rate": 6.442557939214125e-06,
-      "loss": 0.0768,
-      "step": 372
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.8634932101277937,
-      "learning_rate": 6.42481222635489e-06,
-      "loss": 0.1102,
-      "step": 373
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.624917843958734,
-      "learning_rate": 6.407046958164387e-06,
-      "loss": 0.1014,
-      "step": 374
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.3310298175257878,
-      "learning_rate": 6.389262378468219e-06,
-      "loss": 0.0828,
-      "step": 375
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.0614121621946018,
-      "learning_rate": 6.371458731357047e-06,
-      "loss": 0.0987,
-      "step": 376
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 2.7534907444752217,
-      "learning_rate": 6.353636261183214e-06,
-      "loss": 0.0884,
-      "step": 377
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.3293066218185532,
-      "learning_rate": 6.335795212557416e-06,
-      "loss": 0.1089,
-      "step": 378
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.8227195971889414,
-      "learning_rate": 6.3179358303453386e-06,
-      "loss": 0.1146,
-      "step": 379
-    },
-    {
-      "epoch": 0.43,
-      "grad_norm": 1.8879414641081136,
-      "learning_rate": 6.300058359664285e-06,
-      "loss": 0.0771,
-      "step": 380
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 2.0221109980415646,
-      "learning_rate": 6.2821630458798236e-06,
-      "loss": 0.113,
-      "step": 381
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 1.9690496853348498,
-      "learning_rate": 6.264250134602415e-06,
-      "loss": 0.1076,
-      "step": 382
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 1.3353353790171576,
-      "learning_rate": 6.246319871684048e-06,
-      "loss": 0.0983,
-      "step": 383
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 0.8840122811422151,
-      "learning_rate": 6.228372503214853e-06,
-      "loss": 0.0784,
-      "step": 384
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 1.9223687815447972,
-      "learning_rate": 6.210408275519735e-06,
-      "loss": 0.1164,
-      "step": 385
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 1.3646046422183316,
-      "learning_rate": 6.192427435154991e-06,
-      "loss": 0.1062,
-      "step": 386
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 1.3010904867560287,
-      "learning_rate": 6.17443022890492e-06,
-      "loss": 0.0733,
-      "step": 387
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 0.9588966018297974,
-      "learning_rate": 6.156416903778443e-06,
-      "loss": 0.0937,
-      "step": 388
-    },
-    {
-      "epoch": 0.44,
-      "grad_norm": 0.8176889337157417,
-      "learning_rate": 6.138387707005711e-06,
-      "loss": 0.0785,
-      "step": 389
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.3798947208822552,
-      "learning_rate": 6.1203428860347085e-06,
-      "loss": 0.1029,
-      "step": 390
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.2627266387294376,
-      "learning_rate": 6.10228268852786e-06,
-      "loss": 0.0853,
-      "step": 391
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.3636634430274013,
-      "learning_rate": 6.084207362358635e-06,
-      "loss": 0.1015,
-      "step": 392
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.7109591987525221,
-      "learning_rate": 6.066117155608136e-06,
-      "loss": 0.1118,
-      "step": 393
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.517961417989287,
-      "learning_rate": 6.048012316561699e-06,
-      "loss": 0.0852,
-      "step": 394
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 2.342629048761873,
-      "learning_rate": 6.029893093705492e-06,
-      "loss": 0.1187,
-      "step": 395
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 0.9294142148345402,
-      "learning_rate": 6.011759735723096e-06,
-      "loss": 0.0864,
-      "step": 396
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.1950075847793311,
-      "learning_rate": 5.993612491492088e-06,
-      "loss": 0.092,
-      "step": 397
-    },
-    {
-      "epoch": 0.45,
-      "grad_norm": 1.3551545691072298,
-      "learning_rate": 5.975451610080643e-06,
-      "loss": 0.0693,
-      "step": 398
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.216161129705157,
-      "learning_rate": 5.957277340744094e-06,
-      "loss": 0.0895,
-      "step": 399
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.4076300298899016,
-      "learning_rate": 5.939089932921527e-06,
-      "loss": 0.085,
-      "step": 400
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.465518309640694,
-      "learning_rate": 5.920889636232352e-06,
-      "loss": 0.0957,
-      "step": 401
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.3444282299556731,
-      "learning_rate": 5.902676700472874e-06,
-      "loss": 0.1023,
-      "step": 402
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 2.172246680213261,
-      "learning_rate": 5.884451375612865e-06,
-      "loss": 0.0859,
-      "step": 403
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.0096049350842804,
-      "learning_rate": 5.8662139117921435e-06,
-      "loss": 0.0952,
-      "step": 404
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.3924332642607742,
-      "learning_rate": 5.847964559317129e-06,
-      "loss": 0.075,
-      "step": 405
-    },
-    {
-      "epoch": 0.46,
-      "grad_norm": 1.555222595194896,
-      "learning_rate": 5.829703568657406e-06,
-      "loss": 0.1194,
-      "step": 406
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.814069025220444,
-      "learning_rate": 5.8114311904423e-06,
-      "loss": 0.1114,
-      "step": 407
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 2.63589934143905,
-      "learning_rate": 5.793147675457425e-06,
-      "loss": 0.0997,
-      "step": 408
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.672874605967912,
-      "learning_rate": 5.774853274641243e-06,
-      "loss": 0.0936,
-      "step": 409
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.293398379507954,
-      "learning_rate": 5.756548239081626e-06,
-      "loss": 0.0936,
-      "step": 410
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.3509632172781578,
-      "learning_rate": 5.738232820012407e-06,
-      "loss": 0.1014,
-      "step": 411
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.4424133826280734,
-      "learning_rate": 5.719907268809927e-06,
-      "loss": 0.0942,
-      "step": 412
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.6011921811511678,
-      "learning_rate": 5.701571836989591e-06,
-      "loss": 0.0839,
-      "step": 413
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 2.1072468499647248,
-      "learning_rate": 5.683226776202416e-06,
-      "loss": 0.1096,
-      "step": 414
-    },
-    {
-      "epoch": 0.47,
-      "grad_norm": 1.3764568994495563,
-      "learning_rate": 5.664872338231572e-06,
-      "loss": 0.1001,
-      "step": 415
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 2.53101947882973,
-      "learning_rate": 5.64650877498893e-06,
-      "loss": 0.0885,
-      "step": 416
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.9958749325869098,
-      "learning_rate": 5.628136338511607e-06,
-      "loss": 0.1067,
-      "step": 417
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.1068155120562846,
-      "learning_rate": 5.6097552809584996e-06,
-      "loss": 0.0862,
-      "step": 418
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.625671593043311,
-      "learning_rate": 5.591365854606829e-06,
-      "loss": 0.0954,
-      "step": 419
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 0.8652769478201479,
-      "learning_rate": 5.5729683118486785e-06,
-      "loss": 0.0848,
-      "step": 420
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 2.0731466858910217,
-      "learning_rate": 5.554562905187527e-06,
-      "loss": 0.089,
-      "step": 421
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.6660858543441386,
-      "learning_rate": 5.536149887234781e-06,
-      "loss": 0.0982,
-      "step": 422
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.552222550169945,
-      "learning_rate": 5.517729510706316e-06,
-      "loss": 0.1154,
-      "step": 423
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 1.3368802229091306,
-      "learning_rate": 5.499302028418998e-06,
-      "loss": 0.0976,
-      "step": 424
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 2.181126697333057,
-      "learning_rate": 5.480867693287224e-06,
-      "loss": 0.1062,
-      "step": 425
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.9205645782423137,
-      "learning_rate": 5.462426758319439e-06,
-      "loss": 0.0911,
-      "step": 426
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.9632409923514775,
-      "learning_rate": 5.443979476614674e-06,
-      "loss": 0.1095,
-      "step": 427
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.1362611723204445,
-      "learning_rate": 5.425526101359068e-06,
-      "loss": 0.0777,
-      "step": 428
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.3238503828238188,
-      "learning_rate": 5.407066885822391e-06,
-      "loss": 0.0824,
-      "step": 429
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.5813044674729215,
-      "learning_rate": 5.388602083354572e-06,
-      "loss": 0.1091,
-      "step": 430
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.2656866672687868,
-      "learning_rate": 5.370131947382215e-06,
-      "loss": 0.1061,
-      "step": 431
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.3312297948060596,
-      "learning_rate": 5.351656731405132e-06,
-      "loss": 0.0823,
-      "step": 432
-    },
-    {
-      "epoch": 0.49,
-      "grad_norm": 1.4157388997754832,
-      "learning_rate": 5.333176688992856e-06,
-      "loss": 0.0792,
-      "step": 433
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.4094464298882508,
-      "learning_rate": 5.314692073781157e-06,
-      "loss": 0.101,
-      "step": 434
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.4099498232807148,
-      "learning_rate": 5.296203139468572e-06,
-      "loss": 0.0865,
-      "step": 435
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.8502414411111636,
-      "learning_rate": 5.2777101398129135e-06,
-      "loss": 0.0889,
-      "step": 436
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.7560344587273293,
-      "learning_rate": 5.2592133286277925e-06,
-      "loss": 0.0916,
-      "step": 437
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.6710267518378974,
-      "learning_rate": 5.2407129597791295e-06,
-      "loss": 0.1047,
-      "step": 438
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.2201280272734387,
-      "learning_rate": 5.222209287181677e-06,
-      "loss": 0.0891,
-      "step": 439
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 1.4358152308072225,
-      "learning_rate": 5.203702564795528e-06,
-      "loss": 0.0802,
-      "step": 440
-    },
-    {
-      "epoch": 0.5,
-      "grad_norm": 0.8914507091018575,
-      "learning_rate": 5.185193046622636e-06,
-      "loss": 0.0782,
-      "step": 441
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.6165198764693776,
-      "learning_rate": 5.166680986703323e-06,
-      "loss": 0.1246,
-      "step": 442
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.4298748482799117,
-      "learning_rate": 5.148166639112799e-06,
-      "loss": 0.105,
-      "step": 443
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.535559818853895,
-      "learning_rate": 5.129650257957671e-06,
-      "loss": 0.1119,
-      "step": 444
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.9197842071730942,
-      "learning_rate": 5.111132097372459e-06,
-      "loss": 0.1033,
-      "step": 445
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.1599006332138146,
-      "learning_rate": 5.0926124115160976e-06,
-      "loss": 0.0896,
-      "step": 446
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.3974819169152706,
-      "learning_rate": 5.074091454568464e-06,
-      "loss": 0.0969,
-      "step": 447
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 0.9408016749032659,
-      "learning_rate": 5.055569480726876e-06,
-      "loss": 0.0895,
-      "step": 448
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.9952244551309546,
-      "learning_rate": 5.0370467442026115e-06,
-      "loss": 0.0871,
-      "step": 449
-    },
-    {
-      "epoch": 0.51,
-      "grad_norm": 1.36586105745886,
-      "learning_rate": 5.0185234992174114e-06,
-      "loss": 0.0844,
-      "step": 450
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.7479502067142012,
-      "learning_rate": 5e-06,
-      "loss": 0.1044,
-      "step": 451
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.3573048964331627,
-      "learning_rate": 4.981476500782591e-06,
-      "loss": 0.0948,
-      "step": 452
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.473812452772602,
-      "learning_rate": 4.96295325579739e-06,
-      "loss": 0.0797,
-      "step": 453
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 2.0296959916896,
-      "learning_rate": 4.944430519273126e-06,
-      "loss": 0.0895,
-      "step": 454
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.2103951372488675,
-      "learning_rate": 4.925908545431537e-06,
-      "loss": 0.0915,
-      "step": 455
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.434571480202637,
-      "learning_rate": 4.907387588483903e-06,
-      "loss": 0.0962,
-      "step": 456
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.2049199119312812,
-      "learning_rate": 4.8888679026275436e-06,
-      "loss": 0.0909,
-      "step": 457
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.7639545815530557,
-      "learning_rate": 4.870349742042329e-06,
-      "loss": 0.1117,
-      "step": 458
-    },
-    {
-      "epoch": 0.52,
-      "grad_norm": 1.1494030416261993,
-      "learning_rate": 4.8518333608872015e-06,
-      "loss": 0.0752,
-      "step": 459
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.6445773991618207,
-      "learning_rate": 4.8333190132966794e-06,
-      "loss": 0.0861,
-      "step": 460
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.6414731186074902,
-      "learning_rate": 4.814806953377366e-06,
-      "loss": 0.0829,
-      "step": 461
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.8968387997428107,
-      "learning_rate": 4.796297435204473e-06,
-      "loss": 0.0887,
-      "step": 462
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 2.25439526816161,
-      "learning_rate": 4.777790712818324e-06,
-      "loss": 0.075,
-      "step": 463
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.1014332021036057,
-      "learning_rate": 4.759287040220872e-06,
-      "loss": 0.0719,
-      "step": 464
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.2580923566775155,
-      "learning_rate": 4.740786671372209e-06,
-      "loss": 0.0955,
-      "step": 465
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.7123882846797445,
-      "learning_rate": 4.7222898601870864e-06,
-      "loss": 0.0935,
-      "step": 466
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.178774620445043,
-      "learning_rate": 4.703796860531429e-06,
-      "loss": 0.0987,
-      "step": 467
-    },
-    {
-      "epoch": 0.53,
-      "grad_norm": 1.6539278144919904,
-      "learning_rate": 4.6853079262188445e-06,
-      "loss": 0.1124,
-      "step": 468
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.0935968959871103,
-      "learning_rate": 4.666823311007145e-06,
-      "loss": 0.0871,
-      "step": 469
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.3166304048687854,
-      "learning_rate": 4.648343268594869e-06,
-      "loss": 0.0749,
-      "step": 470
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.8561652305018204,
-      "learning_rate": 4.629868052617786e-06,
-      "loss": 0.0847,
-      "step": 471
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.414637359104844,
-      "learning_rate": 4.61139791664543e-06,
-      "loss": 0.0887,
-      "step": 472
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.2510852736375488,
-      "learning_rate": 4.59293311417761e-06,
-      "loss": 0.0713,
-      "step": 473
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.404573593610514,
-      "learning_rate": 4.574473898640933e-06,
-      "loss": 0.0863,
-      "step": 474
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.1678998858209273,
-      "learning_rate": 4.556020523385326e-06,
-      "loss": 0.1042,
-      "step": 475
-    },
-    {
-      "epoch": 0.54,
-      "grad_norm": 1.4292796447931189,
-      "learning_rate": 4.537573241680563e-06,
-      "loss": 0.0898,
-      "step": 476
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.8305004593870045,
-      "learning_rate": 4.519132306712778e-06,
-      "loss": 0.0883,
-      "step": 477
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.706985080173823,
-      "learning_rate": 4.500697971581003e-06,
-      "loss": 0.0628,
-      "step": 478
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.5783162958105084,
-      "learning_rate": 4.482270489293685e-06,
-      "loss": 0.0899,
-      "step": 479
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.2211595747222304,
-      "learning_rate": 4.463850112765221e-06,
-      "loss": 0.0754,
-      "step": 480
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.995983190967539,
-      "learning_rate": 4.445437094812476e-06,
-      "loss": 0.0887,
-      "step": 481
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.428898296054049,
-      "learning_rate": 4.4270316881513215e-06,
-      "loss": 0.1133,
-      "step": 482
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.4225799563091075,
-      "learning_rate": 4.408634145393172e-06,
-      "loss": 0.1184,
-      "step": 483
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.3918275141803165,
-      "learning_rate": 4.390244719041502e-06,
-      "loss": 0.0648,
-      "step": 484
-    },
-    {
-      "epoch": 0.55,
-      "grad_norm": 1.367999116075585,
-      "learning_rate": 4.371863661488394e-06,
-      "loss": 0.0853,
-      "step": 485
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.7079182185930346,
-      "learning_rate": 4.353491225011071e-06,
-      "loss": 0.0949,
-      "step": 486
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.5010142904615251,
-      "learning_rate": 4.335127661768429e-06,
-      "loss": 0.0641,
-      "step": 487
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.7284173282169324,
-      "learning_rate": 4.316773223797585e-06,
-      "loss": 0.0703,
-      "step": 488
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.5621635512490386,
-      "learning_rate": 4.298428163010411e-06,
-      "loss": 0.0869,
-      "step": 489
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.422341370982355,
-      "learning_rate": 4.280092731190075e-06,
-      "loss": 0.0799,
-      "step": 490
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.7501446138446721,
-      "learning_rate": 4.261767179987595e-06,
-      "loss": 0.0897,
-      "step": 491
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 1.2772834345032038,
-      "learning_rate": 4.243451760918376e-06,
-      "loss": 0.1065,
-      "step": 492
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 0.7832782732498063,
-      "learning_rate": 4.225146725358759e-06,
-      "loss": 0.0806,
-      "step": 493
-    },
-    {
-      "epoch": 0.56,
-      "grad_norm": 0.9590851368962775,
-      "learning_rate": 4.206852324542578e-06,
-      "loss": 0.0834,
-      "step": 494
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.5291845240945432,
-      "learning_rate": 4.188568809557701e-06,
-      "loss": 0.0935,
-      "step": 495
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.774265127813176,
-      "learning_rate": 4.170296431342595e-06,
-      "loss": 0.0776,
-      "step": 496
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.6751404930422793,
-      "learning_rate": 4.152035440682874e-06,
-      "loss": 0.1102,
-      "step": 497
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.1017368315986533,
-      "learning_rate": 4.1337860882078564e-06,
-      "loss": 0.0842,
-      "step": 498
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.0095647614965828,
-      "learning_rate": 4.115548624387136e-06,
-      "loss": 0.1033,
-      "step": 499
-    },
-    {
-      "epoch": 0.57,
-      "grad_norm": 1.1850618393493437,
-      "learning_rate": 4.097323299527129e-06,
-      "loss": 0.0935,
-      "step": 500
-    }
-  ],
-  "logging_steps": 1.0,
-  "max_steps": 875,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
-  "save_steps": 500,
-  "total_flos": 27948467372032.0,
-  "train_batch_size": 1,
-  "trial_name": null,
-  "trial_params": null
-}

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ebfe8bb420da2eb16d19fd06f47558897a5e52c061378a3e71ac7060768cf078
-size 7864

llavanext-google_siglip-so400m-patch14-384-Qwen_Qwen2-7B-Instruct-ov_AG_v5_3_split06_fulltune/checkpoint-500/zero_to_fp32.py

@@ -1,604 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) Microsoft Corporation.
-# SPDX-License-Identifier: Apache-2.0
-
-# DeepSpeed Team
-
-# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
-# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
-# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
-# application.
-#
-# example: python zero_to_fp32.py . pytorch_model.bin
-
-import argparse
-import torch
-import glob
-import math
-import os
-import re
-from collections import OrderedDict
-from dataclasses import dataclass
-
-# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
-# DeepSpeed data structures it has to be available in the current python environment.
-from deepspeed.utils import logger
-from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
-                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
-                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
-
-
-@dataclass
-class zero_model_state:
-    buffers: dict()
-    param_shapes: dict()
-    shared_params: list
-    ds_version: int
-    frozen_param_shapes: dict()
-    frozen_param_fragments: dict()
-
-
-debug = 0
-
-# load to cpu
-device = torch.device('cpu')
-
-
-def atoi(text):
-    return int(text) if text.isdigit() else text
-
-
-def natural_keys(text):
-    '''
-    alist.sort(key=natural_keys) sorts in human order
-    http://nedbatchelder.com/blog/200712/human_sorting.html
-    (See Toothy's implementation in the comments)
-    '''
-    return [atoi(c) for c in re.split(r'(\d+)', text)]
-
-
-def get_model_state_file(checkpoint_dir, zero_stage):
-    if not os.path.isdir(checkpoint_dir):
-        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
-
-    # there should be only one file
-    if zero_stage <= 2:
-        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
-    elif zero_stage == 3:
-        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
-
-    if not os.path.exists(file):
-        raise FileNotFoundError(f"can't find model states file at '{file}'")
-
-    return file
-
-
-def get_checkpoint_files(checkpoint_dir, glob_pattern):
-    # XXX: need to test that this simple glob rule works for multi-node setup too
-    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
-
-    if len(ckpt_files) == 0:
-        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
-
-    return ckpt_files
-
-
-def get_optim_files(checkpoint_dir):
-    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
-
-
-def get_model_state_files(checkpoint_dir):
-    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
-
-
-def parse_model_states(files):
-    zero_model_states = []
-    for file in files:
-        state_dict = torch.load(file, map_location=device)
-
-        if BUFFER_NAMES not in state_dict:
-            raise ValueError(f"{file} is not a model state checkpoint")
-        buffer_names = state_dict[BUFFER_NAMES]
-        if debug:
-            print("Found buffers:", buffer_names)
-
-        # recover just the buffers while restoring them to fp32 if they were saved in fp16
-        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
-        param_shapes = state_dict[PARAM_SHAPES]
-
-        # collect parameters that are included in param_shapes
-        param_names = []
-        for s in param_shapes:
-            for name in s.keys():
-                param_names.append(name)
-
-        # update with frozen parameters
-        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
-        if frozen_param_shapes is not None:
-            if debug:
-                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
-            param_names += list(frozen_param_shapes.keys())
-
-        # handle shared params
-        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
-
-        ds_version = state_dict.get(DS_VERSION, None)
-
-        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
-
-        z_model_state = zero_model_state(buffers=buffers,
-                                         param_shapes=param_shapes,
-                                         shared_params=shared_params,
-                                         ds_version=ds_version,
-                                         frozen_param_shapes=frozen_param_shapes,
-                                         frozen_param_fragments=frozen_param_fragments)
-        zero_model_states.append(z_model_state)
-
-    return zero_model_states
-
-
-def parse_optim_states(files, ds_checkpoint_dir):
-
-    total_files = len(files)
-    state_dicts = []
-    for f in files:
-        state_dict = torch.load(f, map_location=device)
-        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
-        # and also handle the case where it was already removed by another helper script
-        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
-        state_dicts.append(state_dict)
-
-    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
-        raise ValueError(f"{files[0]} is not a zero checkpoint")
-    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
-    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
-
-    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
-    # parameters can be different from data parallelism for non-expert parameters. So we can just
-    # use the max of the partition_count to get the dp world_size.
-
-    if type(world_size) is list:
-        world_size = max(world_size)
-
-    if world_size != total_files:
-        raise ValueError(
-            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
-            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
-        )
-
-    # the groups are named differently in each stage
-    if zero_stage <= 2:
-        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
-    elif zero_stage == 3:
-        fp32_groups_key = FP32_FLAT_GROUPS
-    else:
-        raise ValueError(f"unknown zero stage {zero_stage}")
-
-    if zero_stage <= 2:
-        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
-    elif zero_stage == 3:
-        # if there is more than one param group, there will be multiple flattened tensors - one
-        # flattened tensor per group - for simplicity merge them into a single tensor
-        #
-        # XXX: could make the script more memory efficient for when there are multiple groups - it
-        # will require matching the sub-lists of param_shapes for each param group flattened tensor
-
-        fp32_flat_groups = [
-            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
-        ]
-
-    return zero_stage, world_size, fp32_flat_groups
-
-
-def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
-    """
-    Returns fp32 state_dict reconstructed from ds checkpoint
-
-    Args:
-        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
-
-    """
-    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
-
-    optim_files = get_optim_files(ds_checkpoint_dir)
-    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
-    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
-
-    model_files = get_model_state_files(ds_checkpoint_dir)
-
-    zero_model_states = parse_model_states(model_files)
-    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
-
-    if zero_stage <= 2:
-        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                                          exclude_frozen_parameters)
-    elif zero_stage == 3:
-        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                                          exclude_frozen_parameters)
-
-
-def _zero2_merge_frozen_params(state_dict, zero_model_states):
-    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
-        return
-
-    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
-    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
-
-    if debug:
-        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
-        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
-
-        wanted_params = len(frozen_param_shapes)
-        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
-        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
-        print(f'Frozen params: Have {avail_numel} numels to process.')
-        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
-
-    total_params = 0
-    total_numel = 0
-    for name, shape in frozen_param_shapes.items():
-        total_params += 1
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-
-        state_dict[name] = frozen_param_fragments[name]
-
-        if debug:
-            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
-
-    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _has_callable(obj, fn):
-    attr = getattr(obj, fn, None)
-    return callable(attr)
-
-
-def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
-    param_shapes = zero_model_states[0].param_shapes
-
-    # Reconstruction protocol:
-    #
-    # XXX: document this
-
-    if debug:
-        for i in range(world_size):
-            for j in range(len(fp32_flat_groups[0])):
-                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
-
-    # XXX: memory usage doubles here (zero2)
-    num_param_groups = len(fp32_flat_groups[0])
-    merged_single_partition_of_fp32_groups = []
-    for i in range(num_param_groups):
-        merged_partitions = [sd[i] for sd in fp32_flat_groups]
-        full_single_fp32_vector = torch.cat(merged_partitions, 0)
-        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
-    avail_numel = sum(
-        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
-
-    if debug:
-        wanted_params = sum([len(shapes) for shapes in param_shapes])
-        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
-        # not asserting if there is a mismatch due to possible padding
-        print(f"Have {avail_numel} numels to process.")
-        print(f"Need {wanted_numel} numels in {wanted_params} params.")
-
-    # params
-    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
-    # out-of-core computing solution
-    total_numel = 0
-    total_params = 0
-    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
-        offset = 0
-        avail_numel = full_single_fp32_vector.numel()
-        for name, shape in shapes.items():
-
-            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
-            total_numel += unpartitioned_numel
-            total_params += 1
-
-            if debug:
-                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
-            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
-            offset += unpartitioned_numel
-
-        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
-        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
-        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
-        # live optimizer object, so we are checking that the numbers are within the right range
-        align_to = 2 * world_size
-
-        def zero2_align(x):
-            return align_to * math.ceil(x / align_to)
-
-        if debug:
-            print(f"original offset={offset}, avail_numel={avail_numel}")
-
-        offset = zero2_align(offset)
-        avail_numel = zero2_align(avail_numel)
-
-        if debug:
-            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
-
-        # Sanity check
-        if offset != avail_numel:
-            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
-
-    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                               exclude_frozen_parameters):
-    state_dict = OrderedDict()
-
-    # buffers
-    buffers = zero_model_states[0].buffers
-    state_dict.update(buffers)
-    if debug:
-        print(f"added {len(buffers)} buffers")
-
-    if not exclude_frozen_parameters:
-        _zero2_merge_frozen_params(state_dict, zero_model_states)
-
-    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
-
-    # recover shared parameters
-    for pair in zero_model_states[0].shared_params:
-        if pair[1] in state_dict:
-            state_dict[pair[0]] = state_dict[pair[1]]
-
-    return state_dict
-
-
-def zero3_partitioned_param_info(unpartitioned_numel, world_size):
-    remainder = unpartitioned_numel % world_size
-    padding_numel = (world_size - remainder) if remainder else 0
-    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
-    return partitioned_numel, padding_numel
-
-
-def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
-    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
-        return
-
-    if debug:
-        for i in range(world_size):
-            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
-            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
-
-        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
-        wanted_params = len(frozen_param_shapes)
-        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
-        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
-        print(f'Frozen params: Have {avail_numel} numels to process.')
-        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
-
-    total_params = 0
-    total_numel = 0
-    for name, shape in zero_model_states[0].frozen_param_shapes.items():
-        total_params += 1
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-
-        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
-        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
-
-        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
-
-        if debug:
-            print(
-                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
-            )
-
-    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
-    param_shapes = zero_model_states[0].param_shapes
-    avail_numel = fp32_flat_groups[0].numel() * world_size
-    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
-    # param, re-consolidating each param, while dealing with padding if any
-
-    # merge list of dicts, preserving order
-    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
-
-    if debug:
-        for i in range(world_size):
-            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
-
-        wanted_params = len(param_shapes)
-        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
-        # not asserting if there is a mismatch due to possible padding
-        avail_numel = fp32_flat_groups[0].numel() * world_size
-        print(f"Trainable params: Have {avail_numel} numels to process.")
-        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
-
-    # params
-    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
-    # out-of-core computing solution
-    offset = 0
-    total_numel = 0
-    total_params = 0
-    for name, shape in param_shapes.items():
-
-        unpartitioned_numel = shape.numel()
-        total_numel += unpartitioned_numel
-        total_params += 1
-
-        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
-
-        if debug:
-            print(
-                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
-            )
-
-        # XXX: memory usage doubles here
-        state_dict[name] = torch.cat(
-            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
-            0).narrow(0, 0, unpartitioned_numel).view(shape)
-        offset += partitioned_numel
-
-    offset *= world_size
-
-    # Sanity check
-    if offset != avail_numel:
-        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
-
-    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
-
-
-def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
-                                               exclude_frozen_parameters):
-    state_dict = OrderedDict()
-
-    # buffers
-    buffers = zero_model_states[0].buffers
-    state_dict.update(buffers)
-    if debug:
-        print(f"added {len(buffers)} buffers")
-
-    if not exclude_frozen_parameters:
-        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
-
-    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
-
-    # recover shared parameters
-    for pair in zero_model_states[0].shared_params:
-        if pair[1] in state_dict:
-            state_dict[pair[0]] = state_dict[pair[1]]
-
-    return state_dict
-
-
-def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
-    """
-    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
-    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
-    via a model hub.
-
-    Args:
-        - ``checkpoint_dir``: path to the desired checkpoint folder
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
-        - ``exclude_frozen_parameters``: exclude frozen parameters
-
-    Returns:
-        - pytorch ``state_dict``
-
-    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
-    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
-    the checkpoint.
-
-    A typical usage might be ::
-
-        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
-        # do the training and checkpoint saving
-        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
-        model = model.cpu() # move to cpu
-        model.load_state_dict(state_dict)
-        # submit to model hub or save the model to share with others
-
-    In this example the ``model`` will no longer be usable in the deepspeed context of the same
-    application. i.e. you will need to re-initialize the deepspeed engine, since
-    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
-
-    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
-
-    """
-    if tag is None:
-        latest_path = os.path.join(checkpoint_dir, 'latest')
-        if os.path.isfile(latest_path):
-            with open(latest_path, 'r') as fd:
-                tag = fd.read().strip()
-        else:
-            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
-
-    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
-
-    if not os.path.isdir(ds_checkpoint_dir):
-        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
-
-    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
-
-
-def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
-    """
-    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
-    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
-
-    Args:
-        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
-        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
-        - ``exclude_frozen_parameters``: exclude frozen parameters
-    """
-
-    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
-    print(f"Saving fp32 state dict to {output_file}")
-    torch.save(state_dict, output_file)
-
-
-def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
-    """
-    1. Put the provided model to cpu
-    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
-    3. Load it into the provided model
-
-    Args:
-        - ``model``: the model object to update
-        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
-        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
-
-    Returns:
-        - ``model`: modified model
-
-    Make sure you have plenty of CPU memory available before you call this function. If you don't
-    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
-    conveniently placed for you in the checkpoint folder.
-
-    A typical usage might be ::
-
-        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
-        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
-        # submit to model hub or save the model to share with others
-
-    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
-    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
-    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
-
-    """
-    logger.info(f"Extracting fp32 weights")
-    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
-
-    logger.info(f"Overwriting model with fp32 weights")
-    model = model.cpu()
-    model.load_state_dict(state_dict, strict=False)
-
-    return model
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("checkpoint_dir",
-                        type=str,
-                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
-    parser.add_argument(
-        "output_file",
-        type=str,
-        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
-    parser.add_argument("-t",
-                        "--tag",
-                        type=str,
-                        default=None,
-                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
-    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
-    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
-    args = parser.parse_args()
-
-    debug = args.debug
-
-    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
-                                               args.output_file,
-                                               tag=args.tag,
-                                               exclude_frozen_parameters=args.exclude_frozen_parameters)