Regarding the problem of starcoderbase training, the reasoning becomes slower after training
I use the following script to train starcoderbase. I don’t know where there is no right operation, which causes the model reasoning after training to become very slow, and also throws a warning ''you have modified the pretrained model configuration to control generation. this is a deprecated startegy to control generation and will be removed soon..."
The script I refer to is "https://github.com/bigcode-project/starcoder/tree/main/chat"
Does anyone know where I went wrong, thanks
The following is a certain line of my json file:{"content":"\/\/ Copyright (c) 2019 The ReBitcoin Core developers\n\/\/ Distributed under the MIT software license, see the accompanying\n\/\/ file COPYING or http:\/\/www.opensource.org\/licenses\/mit-license.php.\n\n#ifndef REBITCOIN_UTIL_STRING_H\n#define REBITCOIN_UTIL_STRING_H\n\n#include <string>\n#include <vector>\n\n\/**\n * Join a list of items\n *\n * @param list       The list to join\n * @param separator  The separator\n * @param unary_op   Apply this operator to each item in the list\n *\/\ntemplate <typename T, typename UnaryOp>\nstd::string Join(const std::vector<T>& list, const std::string& separator, UnaryOp unary_op)\n{\n    std::string ret;\n    for (size_t i = 0; i < list.size(); ++i) {\n        if (i > 0) ret += separator;\n        ret += unary_op(list.at(i));\n    }\n    return ret;\n}\n\ninline std::string Join(const std::vector[std::string](std::string)& list, const std::string& separator)\n{\n    return Join(list, separator, [](const std::string& i) { return i; });\n}\n\n#endif \/\/ REBITCOIN_UTIL_STRENCODINGS_H\n","avg_line_length":28.0285714286,"max_line_length":92,"alphanum_fraction":0.6850152905,"path":"src\/util\/string.h","size":981}#!/usr/bin/env python
# coding=utf-8
# Copyright 2023 The BigCode & HuggingFace Inc. teams. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Script to instruction fine-tune causal language models on a Hub dataset

Adapted from huggingface/transformers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py
"""

import logging
import math
import os
import random
import sys
from itertools import chain

import torch
import datasets
import transformers
from datasets import load_dataset
from transformers import (AutoModelForCausalLM, AutoTokenizer, Trainer, default_data_collator, set_seed, pipeline)
from transformers.testing_utils import CaptureLogger
from transformers.trainer_utils import get_last_checkpoint
from transformers import DataCollatorWithPadding

from config import DataArguments, ModelArguments, TrainingArguments
from utils import StarChatArgumentParser, REDACTED_TOKEN

logger = logging.getLogger(__name__)


def create_validation_and_test_splits(
        raw_dataset, tokenizer, max_sequence_length=1024
):
    # Get validation, test, and the remaining train dataset
    test_dataset = raw_dataset["test"]
    train_dataset = raw_dataset["train"]

    # Tokenize the train, validation, and test dataset
    tokenized_train_dataset = train_dataset.map(
        lambda x: tokenizer(
            x["content"], truncation=True, padding="max_length", max_length=max_sequence_length
        ),
        batched=True,
        remove_columns=["content"],
    )
    tokenized_test_dataset = test_dataset.map(
        lambda x: tokenizer(
            x["content"], truncation=True, padding="max_length", max_length=max_sequence_length
        ),
        batched=True,
        remove_columns=["content"],
    )

    return tokenized_train_dataset, tokenized_test_dataset


def addline(examples):
    labels = examples["input_ids"].copy()
    examples["labels"] = labels
    return examples


def main():
    parser = StarChatArgumentParser((ModelArguments, DataArguments, TrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".yaml"):
        # If we pass only one argument to the script and it's the path to a YAML file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_yaml_file(os.path.abspath(sys.argv[1]))
    # parse command line args and yaml file
    elif len(sys.argv) > 2 and sys.argv[1].endswith(".yaml"):
        model_args, data_args, training_args = parser.parse_yaml_and_args(os.path.abspath(sys.argv[1]), sys.argv[2:])
    # parse command line args only
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    # Set seed for reproducibility
    set_seed(training_args.seed)

    ###############
    # Setup logging
    ###############
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )
    log_level = training_args.get_process_log_level()
    logger.setLevel(log_level)
    datasets.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.enable_default_handler()
    transformers.utils.logging.enable_explicit_format()

    # Log on each process a small summary
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f" distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
    )
    logger.info(f"Model parameters {model_args}")
    logger.info(f"Data parameters {data_args}")
    logger.info(f"Training/evaluation parameters {training_args}")

    # Login to HuggingFace Hub if needed
    REDACTED_TOKEN()

    ###########################
    # Detecting last checkpoint
    ###########################
    last_checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        last_checkpoint = get_last_checkpoint(training_args.output_dir)
        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
            logger.info(
                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    ###############
    # Load datasets
    ###############
    raw_datasets = load_dataset("json", data_files=data_args.dataset_name, split="train")
    raw_datasets = raw_datasets.train_test_split(test_size=0.005)
    logger.info(
        f"Training on the following datasets and their proportions: {[split + ' : ' + str(dset.num_rows) for split, dset in raw_datasets.items()]}"
    )
    with training_args.main_process_first(desc="Log a few random samples from the raw training set"):
        for index in random.sample(range(len(raw_datasets["train"])), 3):
            logger.info(f"Sample {index} of the raw training set:\n\n{raw_datasets['train'][index]['content']}")

    #####################################
    # Load tokenizer and process datasets
    #####################################
    tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path)
    if tokenizer.pad_token is None:
        tokenizer.pad_token = tokenizer.eos_token

    (train_dataset, eval_dataset) = create_validation_and_test_splits(raw_datasets, tokenizer)
    unused_column = ["avg_line_length", "max_line_length", "alphanum_fraction", "path", "size"]
    train_dataset = train_dataset.map(addline, batched=True,
                                      num_proc=data_args.preprocessing_num_workers,
                                      remove_columns=unused_column)
    eval_dataset = eval_dataset.map(addline, batched=True,
                                    num_proc=data_args.preprocessing_num_workers,
                                    remove_columns=unused_column)

    if training_args.do_train:
        if data_args.max_train_samples is not None:
            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
            train_dataset = train_dataset.select(range(max_train_samples))

    if training_args.do_eval:
        if data_args.max_eval_samples is not None:
            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
            eval_dataset = eval_dataset.select(range(max_eval_samples))

    #######################
    # Load pretrained model
    #######################
    logger.info("*** Load pretrained model ***")
    torch_dtype = (
        model_args.torch_dtype if model_args.torch_dtype in ["auto", None] else getattr(torch, model_args.torch_dtype)
    )
    model = AutoModelForCausalLM.from_pretrained(
        model_args.model_name_or_path,
        revision=model_args.model_revision,
        torch_dtype=torch_dtype,
        use_cache=False if training_args.gradient_checkpointing else True,
    )
    model.resize_token_embeddings(len(tokenizer))

    ########################
    # create a data collator for ?
    ########################
    data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding="max_length", pad_to_multiple_of=8)

    ########################
    # Initialize the Trainer
    ########################
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset if training_args.do_train else None,
        eval_dataset=eval_dataset if training_args.do_eval else None,
        tokenizer=tokenizer,
        # Data collator defaults to DataCollatorWithPadding, so we change it
        # since we've already chunked our corpus
        # data_collator=default_data_collator,
        data_collator=data_collator,
    )

    ###############
    # Training loop
    ###############
    if training_args.do_train:
        logger.info("*** Train ***")
        checkpoint = None
        if training_args.resume_from_checkpoint is not None:
            checkpoint = training_args.resume_from_checkpoint
        elif last_checkpoint is not None:
            checkpoint = last_checkpoint
        train_result = trainer.train(resume_from_checkpoint=checkpoint)

        metrics = train_result.metrics

        max_train_samples = (
            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
        )
        metrics["train_samples"] = min(max_train_samples, len(train_dataset))

        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

    ##########
    # Evaluate
    ##########
    if training_args.do_eval:
        logger.info("*** Evaluate ***")

        metrics = trainer.evaluate()

        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
        try:
            perplexity = math.exp(metrics["eval_loss"])
        except OverflowError:
            perplexity = float("inf")
        metrics["perplexity"] = perplexity

        trainer.log_metrics("eval", metrics)
        trainer.save_metrics("eval", metrics)

    #################################
    # Create model card & push to Hub
    #################################
    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
    if data_args.dataset_name is not None:
        kwargs["dataset_tags"] = data_args.dataset_name
        # if data_args.dataset_config_name is not None:
        #     kwargs["dataset_args"] = data_args.dataset_config_name
        #     kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
        # else:
        #     kwargs["dataset"] = data_args.dataset_name
        #     kwargs["dataset_args"] = "default"
        kwargs["dataset"] = data_args.dataset_name
        kwargs["dataset_args"] = "default"

    # Store dialogue template so we can load it at deployment time
    dialogue_template.save_pretrained(training_args.output_dir)

    if training_args.push_to_hub:
        trainer.push_to_hub(**kwargs)
    else:
        trainer.save_model(training_args.output_dir)
        trainer.create_model_card(**kwargs)

    with training_args.main_process_first(desc="Generate a sample from the model"):
        inputs = "static size_t chrtos(char *buf, size_t size, char byte) {"
        inputs = tokenizer.encode(inputs, return_tensor="pt").to(training_args.device)
        outputs = model.generate(
            **inputs,
            max_new_tokens=256,
            pad_token_id=tokenizer.pad_token_id,
            eos_token_id=tokenizer.eos_token_id,
        )
        logger.info(f"=== SAMPLE OUTPUT ==\n\n{tokenizer.decode(outputs[0], skip_special_tokens=True)}")


if __name__ == "__main__":
    main()
2024-01-25T19:27:03Z
Hello. Please share scripts for training the model. Very necessary.Thank you.