Upload config.yaml with huggingface_hub

config.yaml

@@ -0,0 +1,87 @@
+_config_type: haystacks.embeddings.train_batch.TrainConfig
+accelerator: gpu
+accumulate_grad_batches: 1
+adam_beta1: 0.9
+adam_beta2: 0.95
+base_save_dir: /home/sabri/code/haystacks/checkpoints
+check_val_every_n_epoch: null
+ckpt_path: null
+devices: 8
+dtype: bfloat16
+embedding_module:
+  _config_type: haystacks.embeddings.modeling.TokenEmbeddingModuleConfig
+  batch_attention_layers: []
+  embedding_dim: 8192
+  kwargs: {}
+  target:
+    _is_type: true
+    name: haystacks.embeddings.modeling.TokenEmbeddingModule
+enable_checkpointing: false
+foreach: null
+gradient_clip_val: null
+launch_id: null
+learnable_bias: true
+learnable_temp: true
+limit_train_batches: 1.0
+limit_val_batches: 1.0
+load_hub: true
+log_every_n_steps: 4
+log_grad_norms: false
+loss_comparison: matched
+loss_token_idxs:
+- 64
+- 128
+- 256
+- 512
+- 1024
+lr: 0.0001
+lr_scheduler: null
+manual_save_epochs: null
+manual_save_steps: 8192
+max_epochs: 512
+max_hidden_layers: null
+max_problems: null
+max_seq_len: 1024
+max_steps: -1
+model_name: meta-llama/Llama-3.2-1B-Instruct
+name: no_batch-attention-lr0.0001-bs32-d8192-new
+num_sanity_val_steps: null
+num_workers: 0
+objective: cross_entropy
+output_dir: null
+overfit_batches: 0.0
+precision: bf16
+reload_dataloaders_every_n_epochs: 0
+run_dir: null
+run_id: null
+samples_per_batch: 32
+save_intermediates: true
+script_id: null
+seed: 42
+train_batch_size: 1
+train_data_path: ScalingIntelligence/math-train-l3.2-3Bi-meta-n128
+use_wandb: false
+val_batch_size: 1
+val_check_interval: 64
+val_data_path: ScalingIntelligence/math-test-l3.2-3Bi-meta-n128
+val_rollout_data_path: null
+val_samples_per_batch: 32
+validate_before_train: true
+wandb:
+  _config_type: haystacks.embeddings.train_batch.WandbLoggerConfig
+  group: ''
+  id: null
+  job_type: train
+  kwargs: {}
+  log_model: false
+  mode: online
+  name: null
+  prefix: ''
+  project: haystacks
+  save_dir: .
+  tags: []
+  target:
+    _is_type: true
+    name: pytorch_lightning.loggers.wandb.WandbLogger
+weight_decay: 0.1
+weights_only: true