Upload 13 files

README.md

@@ -0,0 +1,42 @@
+---
+library_name: peft
+base_model: Qwen/Qwen-7B-Chat
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- _load_in_8bit: False
+- _load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+- bnb_4bit_quant_storage: uint8
+- load_in_4bit: True
+- load_in_8bit: False
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- _load_in_8bit: False
+- _load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+- bnb_4bit_quant_storage: uint8
+- load_in_4bit: True
+- load_in_8bit: False
+### Framework versions
+
+- PEFT 0.10.0
+- PEFT 0.4.0
+
+- PEFT 0.4.0

adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen-7B-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn",
+    "c_proj",
+    "w1",
+    "w2"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28a0a588fd3bbdc025a5f0219a67e55dfc86b7c52ccf9d687691eb4530beebdf
+size 572564296

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.97,
+    "total_flos": 1423859773341696.0,
+    "train_loss": 0.806224524974823,
+    "train_runtime": 27.1159,
+    "train_samples_per_second": 1.217,
+    "train_steps_per_second": 0.074
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

tokenization_qwen.py

@@ -0,0 +1,276 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union
+
+import tiktoken
+from transformers import PreTrainedTokenizer, AddedToken
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+# changed to use actual index to avoid misconfiguration with vocabulary expansion
+SPECIAL_START_ID = 151643
+SPECIAL_TOKENS = tuple(
+    enumerate(
+        (
+            (
+                ENDOFTEXT,
+                IMSTART,
+                IMEND,
+            )
+            + EXTRAS
+        ),
+        start=SPECIAL_START_ID,
+    )
+)
+SPECIAL_TOKENS_SET = set(t for i, t in SPECIAL_TOKENS)
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        extra_vocab_file=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # how to handle errors in decoding UTF-8 byte sequences
+        # use ignore if you are in streaming inference
+        self.errors = errors  
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: Dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in SPECIAL_TOKENS
+        }
+
+        # try load extra vocab from file
+        if extra_vocab_file is not None:
+            used_ids = set(self.mergeable_ranks.values()) | set(self.special_tokens.values())
+            extra_mergeable_ranks = _load_tiktoken_bpe(extra_vocab_file)
+            for token, index in extra_mergeable_ranks.items():
+                if token in self.mergeable_ranks:
+                    logger.info(f"extra token {token} exists, skipping")
+                    continue
+                if index in used_ids:
+                    logger.info(f'the index {index} for extra token {token} exists, skipping')
+                    continue
+                self.mergeable_ranks[token] = index
+            # the index may be sparse after this, but don't worry tiktoken.Encoding will handle this
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __getstate__(self):
+        # for pickle lovers
+        state = self.__dict__.copy()
+        del state["tokenizer"]
+        return state
+
+    def __setstate__(self, state):
+        # tokenizer is not python native; don't pass it; rebuild it
+        self.__dict__.update(state)
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        self.tokenizer = enc
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(
+        self,
+        new_tokens: Union[List[str], List[AddedToken]],
+        special_tokens: bool = False,
+    ) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError("Adding regular tokens is not supported")
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS_SET:
+                raise ValueError("Adding unknown special tokens is not supported")
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "QWenTokenizer"
+}

train.log

@@ -0,0 +1,2409 @@
+2024-05-21 13:11:53.195 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_13-11-52_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 13:11:53.197 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 13:12:44.163 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_13-12-43_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 13:12:44.166 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 13:12:44.696 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 13:12:44.697 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 13:12:44.697 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 13:13:28.400 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_13-13-28_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 13:13:28.403 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 13:13:28.848 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 13:13:28.849 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 13:13:28.849 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 13:13:57.006 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'w1', 'c_attn', 'w2']
+2024-05-21 13:14:01.246 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 13:14:01.253 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 13:14:01.254 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 13:14:01.254 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 13:14:01.254 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 13:14:01.260 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 13:14:01.260 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 13:14:01.303 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 13:14:18.008 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_13-14-17_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 13:14:18.011 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 13:14:18.461 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 13:14:18.461 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 13:14:18.462 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 13:14:30.427 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'w2', 'w1', 'c_attn']
+2024-05-21 13:14:34.822 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 13:14:34.830 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 13:14:34.830 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 13:14:34.830 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 13:14:34.831 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 13:14:34.832 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 13:14:34.832 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 13:14:34.873 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 13:15:24.386 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_13-15-24_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 13:15:24.389 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 13:15:24.833 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 13:15:24.833 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 13:15:24.834 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 13:15:36.611 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w1', 'w2', 'c_proj', 'c_attn']
+2024-05-21 13:15:40.843 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 13:15:40.851 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 13:15:40.851 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 13:15:40.851 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 13:15:40.852 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 13:15:40.853 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 13:15:40.854 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 13:15:40.895 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 15:20:14.959 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_15-20-14_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 15:20:14.962 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 15:20:15.449 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 15:20:15.450 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 15:20:15.450 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 15:20:42.755 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'w1', 'c_attn', 'w2']
+2024-05-21 15:20:47.021 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 15:20:47.029 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 15:20:47.029 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 15:20:47.029 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 15:20:47.029 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 15:20:47.034 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 15:20:47.034 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 15:20:47.077 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 15:37:02.692 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_15-37-02_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 15:37:02.695 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 15:37:03.274 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 15:37:03.274 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 15:37:03.275 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 15:37:31.115 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w2', 'w1', 'c_proj', 'c_attn']
+2024-05-21 15:38:12.051 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 15:38:12.058 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 15:38:12.059 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 15:38:12.059 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 15:38:12.059 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 15:38:12.063 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 15:38:12.063 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 15:38:12.105 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 15:48:05.000 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_15-48-04_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 15:48:05.004 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 15:48:05.571 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 15:48:05.572 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 15:48:05.572 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 15:48:32.241 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_attn', 'w2', 'c_proj', 'w1']
+2024-05-21 15:49:13.600 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 15:49:13.607 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 15:49:13.607 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 15:49:13.607 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 15:49:13.608 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 15:49:13.611 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 15:49:13.612 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 15:49:13.654 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-21 15:57:36.477 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=2,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May21_15-57-36_ts-ccabdee9f774458487b5dd0f562f0b70-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+optim_target_modules=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-21 15:57:36.480 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-21 15:57:37.043 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-21 15:57:37.043 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-21 15:57:37.044 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-21 15:58:02.666 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w2', 'w1', 'c_proj', 'c_attn']
+2024-05-21 15:58:45.333 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-21 15:58:45.341 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-21 15:58:45.341 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-21 15:58:45.341 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-21 15:58:45.341 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-21 15:58:45.346 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-21 15:58:45.346 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-21 15:58:45.387 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 10:33:52.253 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_10-33-52_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 10:33:52.256 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 10:33:52.746 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 10:33:52.747 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 10:33:52.747 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 10:34:24.498 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w1', 'w2', 'c_proj', 'c_attn']
+2024-05-23 10:34:28.849 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 10:34:28.856 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 10:34:28.857 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 10:34:28.857 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 10:34:28.857 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 10:34:28.862 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 10:34:28.862 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 10:34:28.908 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 10:46:55.771 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_10-46-55_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 10:46:55.774 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 10:46:56.246 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 10:46:56.246 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 10:46:56.247 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 10:47:29.916 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w2', 'w1', 'c_attn', 'c_proj']
+2024-05-23 10:47:34.339 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 10:47:34.404 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 10:47:34.405 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 10:47:34.405 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 10:47:34.405 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 10:47:34.411 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 10:47:34.411 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 10:47:34.456 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:03:38.908 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-03-38_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:03:38.911 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:03:39.404 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:03:39.405 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:03:39.405 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:04:06.829 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'c_attn', 'w1', 'w2']
+2024-05-23 11:04:11.229 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:04:11.237 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:04:11.237 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:04:11.238 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:04:11.238 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:04:11.242 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:04:11.242 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:04:11.282 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:09:34.482 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-09-34_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:09:34.485 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:09:34.949 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:09:34.949 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:09:34.950 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:10:02.282 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'c_attn', 'w2', 'w1']
+2024-05-23 11:10:07.418 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:10:07.426 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:10:07.427 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:10:07.427 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:10:07.427 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:10:07.431 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:10:07.431 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:10:07.479 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:18:23.072 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-18-23_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:18:23.076 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:18:23.596 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:18:23.597 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:18:23.597 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:18:50.129 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'w2', 'c_attn', 'w1']
+2024-05-23 11:18:54.414 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:18:54.422 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:18:54.422 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:18:54.422 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:18:54.423 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:18:54.428 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:18:54.428 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:18:54.469 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:23:01.591 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-23-01_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:23:01.594 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:23:02.063 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:23:02.064 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:23:02.064 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:23:29.599 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_attn', 'c_proj', 'w2', 'w1']
+2024-05-23 11:23:33.944 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:23:33.951 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:23:33.952 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:23:33.952 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:23:33.952 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:23:33.957 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:23:33.957 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:23:34.049 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:24:24.214 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-24-24_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:24:24.217 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:24:24.676 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:24:24.677 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:24:24.677 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:24:36.170 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_proj', 'w1', 'c_attn', 'w2']
+2024-05-23 11:24:40.516 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:24:40.524 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:24:40.524 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:24:40.524 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:24:40.524 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:24:40.526 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:24:40.526 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:24:40.570 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:30:05.839 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-30-05_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:30:05.843 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:30:06.357 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:30:06.358 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:30:06.358 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:30:34.001 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['w1', 'w2', 'c_proj', 'c_attn']
+2024-05-23 11:30:38.329 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:30:38.337 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:30:38.337 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:30:38.337 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:30:38.338 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:30:38.342 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:30:38.342 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:30:38.383 | INFO     | __main__:main:387 - *** starting training ***
+2024-05-23 11:58:07.400 | INFO     | __main__:setup_everything:52 - train_args:TrainingArguments(
+_n_gpu=1,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=True,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=16,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0002,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=output/firefly-qwen-7b-sft-qlora/runs/May23_11-58-07_ts-2f5a9417da5f4de29903e0323581dcaa-launcher,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=100,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=constant_with_warmup,
+max_grad_norm=0.3,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1,
+optim=paged_adamw_32bit,
+optim_args=None,
+output_dir=output/firefly-qwen-7b-sft-qlora,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=output/firefly-qwen-7b-sft-qlora,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=100,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=100,
+weight_decay=0,
+)
+2024-05-23 11:58:07.403 | INFO     | __main__:init_components:333 - Initializing components...
+2024-05-23 11:58:07.884 | INFO     | __main__:load_tokenizer:211 - vocab_size of tokenizer: 151851
+2024-05-23 11:58:07.884 | INFO     | __main__:load_model:220 - Loading model from base model: Qwen/Qwen-7B-Chat
+2024-05-23 11:58:07.885 | INFO     | __main__:load_model:221 - Train model with qlora
+2024-05-23 11:58:35.827 | INFO     | __main__:find_all_linear_names:85 - LoRA target module names: ['c_attn', 'c_proj', 'w1', 'w2']
+2024-05-23 11:58:40.220 | INFO     | __main__:load_model:283 - memory footprint of model: 8.189956784248352 GB
+2024-05-23 11:58:40.228 | INFO     | __main__:load_model:295 - Total model params: 4626.45M
+2024-05-23 11:58:40.228 | INFO     | __main__:init_components:349 - Train model with sft task
+2024-05-23 11:58:40.228 | INFO     | __main__:load_sft_dataset:315 - Loading data with UnifiedSFTDataset
+2024-05-23 11:58:40.228 | INFO     | component.dataset:__init__:19 - Loading data: ./data/dummy_data.jsonl
+2024-05-23 11:58:40.232 | INFO     | component.dataset:__init__:22 - Use template "qwen" for training
+2024-05-23 11:58:40.232 | INFO     | component.dataset:__init__:23 - There are 33 data in dataset
+2024-05-23 11:58:40.276 | INFO     | __main__:main:387 - *** starting training ***

train_args.json

@@ -0,0 +1,30 @@
+{
+    "output_dir": "output/firefly-qwen-7b-sft-qlora",
+    "model_name_or_path": "Qwen/Qwen-7B-Chat",
+    "train_file": "./data/dummy_data.jsonl",
+    "template_name": "qwen",
+    "num_train_epochs": 1,
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 16,
+    "learning_rate": 0.0002,
+    "max_seq_length": 1024,
+    "logging_steps": 100,
+    "save_steps": 100,
+    "save_total_limit": 1,
+    "lr_scheduler_type": "constant_with_warmup",
+    "warmup_steps": 100,
+    "lora_rank": 64,
+    "lora_alpha": 16,
+    "lora_dropout": 0.05,
+    "gradient_checkpointing": true,
+    "disable_tqdm": false,
+    "optim": "paged_adamw_32bit",
+    "seed": 42,
+    "fp16": true,
+    "report_to": "tensorboard",
+    "dataloader_num_workers": 0,
+    "save_strategy": "steps",
+    "weight_decay": 0,
+    "max_grad_norm": 0.3,
+    "remove_unused_columns": false
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 0.97,
+    "train_loss": 0.806224524974823,
+    "train_runtime": 27.1159,
+    "train_samples_per_second": 1.217,
+    "train_steps_per_second": 0.074
+}

trainer_state.json

@@ -0,0 +1,30 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9696969696969697,
+  "eval_steps": 500,
+  "global_step": 2,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.97,
+      "step": 2,
+      "total_flos": 1423859773341696.0,
+      "train_loss": 0.806224524974823,
+      "train_runtime": 27.1159,
+      "train_samples_per_second": 1.217,
+      "train_steps_per_second": 0.074
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 2,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 1423859773341696.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0179168c7b29a8565c8721548394850acb0edeb60f5393fc89374150bbc77757
+size 4539