Unexpected Keywork Argument
import torchfrom transformers import WhisperFeatureExtractor, WhisperForConditionalGeneration, WhisperTokenizer, WhisperProcessorfrom peft import get_peft_model, LoraConfig, TaskTypeimport osfrom torch.utils.data import Dataset, DataLoaderimport torchaudioimport pandas as pdLoad Feature extractor, Tokenizer, and Processorfeature_extractor = WhisperFeatureExtractor.from_pretrained(“openai/whisper-small”)tokenizer = WhisperTokenizer.from_pretrained(“openai/whisper-small”, language=“Tamil”, task=“transcribe”)processor = WhisperProcessor.from_pretrained(“openai/whisper-small”, language=“Tamil”, task=“transcribe”)Load the modelmodel = WhisperForConditionalGeneration.from_pretrained(“openai/whisper-small”)Define the target modules for LoRA based on the actual model structuretarget_modules = [# Encoder layers“model.encoder.layers.0.self_attn.k_proj”,“model.encoder.layers.0.self_attn.v_proj”,“model.encoder.layers.0.self_attn.q_proj”,“model.encoder.layers.0.self_attn.out_proj”,“model.encoder.layers.0.fc1”,“model.encoder.layers.0.fc2”,“model.encoder.layers.1.self_attn.k_proj”,“model.encoder.layers.1.self_attn.v_proj”,“model.encoder.layers.1.self_attn.q_proj”,“model.encoder.layers.1.self_attn.out_proj”,“model.encoder.layers.1.fc1”,“model.encoder.layers.1.fc2”,“model.encoder.layers.2.self_attn.k_proj”,“model.encoder.layers.2.self_attn.v_proj”,“model.encoder.layers.2.self_attn.q_proj”,“model.encoder.layers.2.self_attn.out_proj”,“model.encoder.layers.2.fc1”,“model.encoder.layers.2.fc2”,“model.encoder.layers.3.self_attn.k_proj”,“model.encoder.layers.3.self_attn.v_proj”,“model.encoder.layers.3.self_attn.q_proj”,“model.encoder.layers.3.self_attn.out_proj”,“model.encoder.layers.3.fc1”,“model.encoder.layers.3.fc2”,“model.encoder.layers.4.self_attn.k_proj”,“model.encoder.layers.4.self_attn.v_proj”,“model.encoder.layers.4.self_attn.q_proj”,“model.encoder.layers.4.self_attn.out_proj”,“model.encoder.layers.4.fc1”,“model.encoder.layers.4.fc2”,“model.encoder.layers.5.self_attn.k_proj”,“model.encoder.layers.5.self_attn.v_proj”,“model.encoder.layers.5.self_attn.q_proj”,“model.encoder.layers.5.self_attn.out_proj”,“model.encoder.layers.5.fc1”,“model.encoder.layers.5.fc2”,“model.encoder.layers.6.self_attn.k_proj”,“model.encoder.layers.6.self_attn.v_proj”,“model.encoder.layers.6.self_attn.q_proj”,“model.encoder.layers.6.self_attn.out_proj”,“model.encoder.layers.6.fc1”,“model.encoder.layers.6.fc2”,“model.encoder.layers.7.self_attn.k_proj”,“model.encoder.layers.7.self_attn.v_proj”,“model.encoder.layers.7.self_attn.q_proj”,“model.encoder.layers.7.self_attn.out_proj”,“model.encoder.layers.7.fc1”,“model.encoder.layers.7.fc2”,“model.encoder.layers.8.self_attn.k_proj”,“model.encoder.layers.8.self_attn.v_proj”,“model.encoder.layers.8.self_attn.q_proj”,“model.encoder.layers.8.self_attn.out_proj”,“model.encoder.layers.8.fc1”,“model.encoder.layers.8.fc2”,“model.encoder.layers.9.self_attn.k_proj”,“model.encoder.layers.9.self_attn.v_proj”,“model.encoder.layers.9.self_attn.q_proj”,“model.encoder.layers.9.self_attn.out_proj”,“model.encoder.layers.9.fc1”,“model.encoder.layers.9.fc2”,“model.encoder.layers.10.self_attn.k_proj”,“model.encoder.layers.10.self_attn.v_proj”,“model.encoder.layers.10.self_attn.q_proj”,“model.encoder.layers.10.self_attn.out_proj”,“model.encoder.layers.10.fc1”,“model.encoder.layers.10.fc2”,“model.encoder.layers.11.self_attn.k_proj”,“model.encoder.layers.11.self_attn.v_proj”,“model.encoder.layers.11.self_attn.q_proj”,“model.encoder.layers.11.self_attn.out_proj”,“model.encoder.layers.11.fc1”,“model.encoder.layers.11.fc2”,# Decoder layers
"model.decoder.layers.0.self_attn.k_proj",
"model.decoder.layers.0.self_attn.v_proj",
"model.decoder.layers.0.self_attn.q_proj",
"model.decoder.layers.0.self_attn.out_proj",
"model.decoder.layers.0.encoder_attn.k_proj",
"model.decoder.layers.0.encoder_attn.v_proj",
"model.decoder.layers.0.encoder_attn.q_proj",
"model.decoder.layers.0.encoder_attn.out_proj",
"model.decoder.layers.0.fc1",
"model.decoder.layers.0.fc2",
"model.decoder.layers.1.self_attn.k_proj",
"model.decoder.layers.1.self_attn.v_proj",
"model.decoder.layers.1.self_attn.q_proj",
"model.decoder.layers.1.self_attn.out_proj",
"model.decoder.layers.1.encoder_attn.k_proj",
"model.decoder.layers.1.encoder_attn.v_proj",
"model.decoder.layers.1.encoder_attn.q_proj",
"model.decoder.layers.1.encoder_attn.out_proj",
"model.decoder.layers.1.fc1",
"model.decoder.layers.1.fc2",
"model.decoder.layers.2.self_attn.k_proj",
"model.decoder.layers.2.self_attn.v_proj",
"model.decoder.layers.2.self_attn.q_proj",
"model.decoder.layers.2.self_attn.out_proj",
"model.decoder.layers.2.encoder_attn.k_proj",
"model.decoder.layers.2.encoder_attn.v_proj",
"model.decoder.layers.2.encoder_attn.q_proj",
"model.decoder.layers.2.encoder_attn.out_proj",
"model.decoder.layers.2.fc1",
"model.decoder.layers.2.fc2",
"model.decoder.layers.3.self_attn.k_proj",
"model.decoder.layers.3.self_attn.v_proj",
"model.decoder.layers.3.self_attn.q_proj",
"model.decoder.layers.3.self_attn.out_proj",
"model.decoder.layers.3.encoder_attn.k_proj",
"model.decoder.layers.3.encoder_attn.v_proj",
"model.decoder.layers.3.encoder_attn.q_proj",
"model.decoder.layers.3.encoder_attn.out_proj",
"model.decoder.layers.3.fc1",
"model.decoder.layers.3.fc2",
"model.decoder.layers.4.self_attn.k_proj",
"model.decoder.layers.4.self_attn.v_proj",
"model.decoder.layers.4.self_attn.q_proj",
"model.decoder.layers.4.self_attn.out_proj",
"model.decoder.layers.4.encoder_attn.k_proj",
"model.decoder.layers.4.encoder_attn.v_proj",
"model.decoder.layers.4.encoder_attn.q_proj",
"model.decoder.layers.4.encoder_attn.out_proj",
"model.decoder.layers.4.fc1",
"model.decoder.layers.4.fc2",
"model.decoder.layers.5.self_attn.k_proj",
"model.decoder.layers.5.self_attn.v_proj",
"model.decoder.layers.5.self_attn.q_proj",
"model.decoder.layers.5.self_attn.out_proj",
"model.decoder.layers.5.encoder_attn.k_proj",
"model.decoder.layers.5.encoder_attn.v_proj",
"model.decoder.layers.5.encoder_attn.q_proj",
"model.decoder.layers.5.encoder_attn.out_proj",
"model.decoder.layers.5.fc1",
"model.decoder.layers.5.fc2",
"model.decoder.layers.6.self_attn.k_proj",
"model.decoder.layers.6.self_attn.v_proj",
"model.decoder.layers.6.self_attn.q_proj",
"model.decoder.layers.6.self_attn.out_proj",
"model.decoder.layers.6.encoder_attn.k_proj",
"model.decoder.layers.6.encoder_attn.v_proj",
"model.decoder.layers.6.encoder_attn.q_proj",
"model.decoder.layers.6.encoder_attn.out_proj",
"model.decoder.layers.6.fc1",
"model.decoder.layers.6.fc2",
"model.decoder.layers.7.self_attn.k_proj",
"model.decoder.layers.7.self_attn.v_proj",
"model.decoder.layers.7.self_attn.q_proj",
"model.decoder.layers.7.self_attn.out_proj",
"model.decoder.layers.7.encoder_attn.k_proj",
"model.decoder.layers.7.encoder_attn.v_proj",
"model.decoder.layers.7.encoder_attn.q_proj",
"model.decoder.layers.7.encoder_attn.out_proj",
"model.decoder.layers.7.fc1",
"model.decoder.layers.7.fc2",
"model.decoder.layers.8.self_attn.k_proj",
"model.decoder.layers.8.self_attn.v_proj",
"model.decoder.layers.8.self_attn.q_proj",
"model.decoder.layers.8.self_attn.out_proj",
"model.decoder.layers.8.encoder_attn.k_proj",
"model.decoder.layers.8.encoder_attn.v_proj",
"model.decoder.layers.8.encoder_attn.q_proj",
"model.decoder.layers.8.encoder_attn.out_proj",
"model.decoder.layers.8.fc1",
"model.decoder.layers.8.fc2",
"model.decoder.layers.9.self_attn.k_proj",
"model.decoder.layers.9.self_attn.v_proj",
"model.decoder.layers.9.self_attn.q_proj",
"model.decoder.layers.9.self_attn.out_proj",
"model.decoder.layers.9.encoder_attn.k_proj",
"model.decoder.layers.9.encoder_attn.v_proj",
"model.decoder.layers.9.encoder_attn.q_proj",
"model.decoder.layers.9.encoder_attn.out_proj",
"model.decoder.layers.9.fc1",
"model.decoder.layers.9.fc2",
"model.decoder.layers.10.self_attn.k_proj",
"model.decoder.layers.10.self_attn.v_proj",
"model.decoder.layers.10.self_attn.q_proj",
"model.decoder.layers.10.self_attn.out_proj",
"model.decoder.layers.10.encoder_attn.k_proj",
"model.decoder.layers.10.encoder_attn.v_proj",
"model.decoder.layers.10.encoder_attn.q_proj",
"model.decoder.layers.10.encoder_attn.out_proj",
"model.decoder.layers.10.fc1",
"model.decoder.layers.10.fc2",
"model.decoder.layers.11.self_attn.k_proj",
"model.decoder.layers.11.self_attn.v_proj",
"model.decoder.layers.11.self_attn.q_proj",
"model.decoder.layers.11.self_attn.out_proj",
"model.decoder.layers.11.encoder_attn.k_proj",
"model.decoder.layers.11.encoder_attn.v_proj",
"model.decoder.layers.11.encoder_attn.q_proj",
"model.decoder.layers.11.encoder_attn.out_proj",
"model.decoder.layers.11.fc1",
"model.decoder.layers.11.fc2",]Configure LoRAconfig = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM,r=8,lora_alpha=16,target_modules=target_modules,lora_dropout=0.1,)for name, module in model.named_modules():print(name)Apply LoRA to the modellora_model = get_peft_model(model, config)Set model to training modelora_model.train()class WhisperDataset(Dataset):definit(self, csv_file, audio_dir, processor):self.data = pd.read_csv(csv_file)self.audio_dir = audio_dirself.processor = processordef __len__(self):
    return len(self.data)

def __getitem__(self, idx):
    if torch.is_tensor(idx):
        idx = idx.tolist()

    audio_path = f"{self.audio_dir}/{self.data.iloc[idx, 0]}"
    transcription = self.data.iloc[idx, 1]

    # Load audio file
    speech_array, sampling_rate = torchaudio.load(audio_path)

    # Process the audio and transcription
    input_features = self.processor.feature_extractor(speech_array.squeeze().numpy(), sampling_rate=sampling_rate, return_tensors="pt").input_features
    labels = self.processor.tokenizer(transcription, return_tensors="pt").input_ids
    # print(f"Processing index: {idx}")
    # print(f"Transcription: {transcription}")
    # print(f"Input features shape: {input_features.shape}")
    # print(f"Labels (input_ids): {labels}")

    return {
        "input_features": input_features.squeeze(),
        "labels": labels.squeeze()
    }Define a custom collate function to pad sequencesdef collate_fn(batch):input_features = [item[‘input_features’] for item in batch]labels = [item[‘labels’] for item in batch]# Pad sequences
input_features_padded = torch.nn.utils.rnn.pad_sequence(input_features, batch_first=True, padding_value=0)
labels_padded = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=tokenizer.pad_token_id)

return {
    'input_features': input_features_padded,
    'labels': labels_padded
}Example usagecsv_file = “./dup.csv”  # Path to your CSV fileaudio_dir = “.”  # Path to the directory containing your audio filesprocessor = WhisperProcessor.from_pretrained(“openai/whisper-small”, language=“Tamil”, task=“transcribe”)dataset = WhisperDataset(csv_file, audio_dir, processor)dataloader = DataLoader(dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)Define the optimizeroptimizer = torch.optim.Adam(lora_model.parameters(), lr=1e-4)Fine-tuning loopnum_epochs = 3  # Define the number of epochsfor epoch in range(num_epochs):for batch in dataloader:# Forward passinput_features = batch[‘input_features’].to(lora_model.device)labels = batch[‘labels’].to(lora_model.device)print(f"Batch input features shape: {input_features.shape}")
    print(f"Batch labels shape: {labels.shape}")
    print(f"Batch labels: {labels}")
    # print(input_features.shape)
    outputs = lora_model(input_features=input_features, labels=labels)
    loss = outputs.loss

    # Backward pass
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {loss.item()}")Save the fine-tuned modellora_model.save_pretrained(“/content/fine_tuned_whisper_with_lora”)TypeError: WhisperForConditionalGeneration.forward() got an unexpected keyword argument ‘input_ids’Above error is shown for the code
2024-06-19T06:58:42Z
Same here, it seems that Peft wrapper always calls the model with the input_ids and input_embeds kwargs, which is unecpected by the Whisper model
2024-06-19T10:30:25Z
It looks likeget_peft_modelis only used to get the more appropriate model based on argumenttask_typefrom LoraConfig.Since none of the wrappers work for Whisper, you have to create your own implementation.So instead oflora_model = get_peft_model(model, config), you can uselora_model = WhisperTuner(model, config).Where WhisperTuner would be something like this:class WhisperTuner(PeftModel):
    def __init__(self, model: torch.nn.Module, peft_config: PeftConfig, adapter_name: str = "default") -> None:
        super().__init__(model, peft_config, adapter_name)
        self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation
        self.base_model_prepare_encoder_decoder_kwargs_for_generation = (
            self.base_model._prepare_encoder_decoder_kwargs_for_generation
        )

    def forward(
            self,
            attention_mask=None,
            decoder_input_ids=None,
            decoder_attention_mask=None,
            decoder_inputs_embeds=None,
            labels=None,
            output_attentions=None,
            output_hidden_states=None,
            return_dict=None,
            task_ids=None,
            **kwargs,
    ):
        peft_config = self.active_peft_config
        if not peft_config.is_prompt_learning:
            if peft_config.peft_type == PeftType.POLY:
                kwargs["task_ids"] = task_ids

            with self._enable_peft_forward_hooks(**kwargs):
                kwargs = {k: v for k, v in kwargs.items() if k not in self.special_peft_forward_args}
                return self.base_model(
                    attention_mask=attention_mask,
                    decoder_input_ids=decoder_input_ids,
                    decoder_attention_mask=decoder_attention_mask,
                    decoder_inputs_embeds=decoder_inputs_embeds,
                    labels=labels,
                    output_attentions=output_attentions,
                    output_hidden_states=output_hidden_states,
                    return_dict=return_dict,
                    **kwargs,
                )

    def generate(self, **kwargs):
        peft_config = self.active_peft_config
        self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation
        self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
            self._prepare_encoder_decoder_kwargs_for_generation
        )
        try:
            if not peft_config.is_prompt_learning:
                with self._enable_peft_forward_hooks(**kwargs):
                    kwargs = {k: v for k, v in kwargs.items() if k not in self.special_peft_forward_args}
                    outputs = self.base_model.generate(**kwargs)
        except:
            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
            self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
                self.base_model_prepare_encoder_decoder_kwargs_for_generation
            )
            raise
        else:
            self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation
            self.base_model._prepare_encoder_decoder_kwargs_for_generation = (
                self.base_model_prepare_encoder_decoder_kwargs_for_generation
            )
            return outputs

    def prepare_inputs_for_generation(self, *args, task_ids: torch.Tensor = None, **kwargs):
        peft_config = self.active_peft_config
        model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs)
        if peft_config.peft_type == PeftType.POLY:
            model_kwargs["task_ids"] = task_ids
        if model_kwargs["past_key_values"] is None and peft_config.peft_type == PeftType.PREFIX_TUNING:
            batch_size = model_kwargs["decoder_input_ids"].shape[0]
            past_key_values = self.get_prompt(batch_size)
            model_kwargs["past_key_values"] = past_key_values

        return model_kwargs(I just copied an implementation of PeftModel and deleted all paths and entries that are not needed for whisper, there’s probably room for improvement in this code)