Finetuning CLIP model raises IndexError: index out of range in self
I’m trying to finetune the CLIP model for my private dataset. I have written CustomDataset for that, the data loading is working well but when I try to train the model with Trainer, It raises IndexError: index out of range in selfCustom Dataset​```class ClassifierDataset(Dataset):definit(self, processor, folds: int, mode=“train”):super(ClassifierDataset, self).init()self.mode = mode
    self.fold = folds
    self.processor = processor
    self.max_target_length = 32
    
    self.train_transform = A.Compose(
        [
            A.CLAHE(),
            A.RandomRotate90(),
            A.Transpose(),
            A.ShiftScaleRotate(
                 shift_limit=0.0625, scale_limit=0.50, rotate_limit=45, p=0.75
             ),
            A.Blur(blur_limit=3),
            A.OpticalDistortion(),
            A.GridDistortion(),
            A.HueSaturationValue(),
            A.VerticalFlip(),
            A.HorizontalFlip(),
            A.Normalize(),
            ToTensorV2(),
        ]
    )

    self.val_transform = A.Compose(
        [

            A.Normalize(),
            ToTensorV2(),
        ]
    )
    
    
    self.df = pd.read_csv(str(Path.cwd() / "data_new.csv"))

    self.train_df = self.df[self.df.folds != self.fold].reset_index(drop=True)
    self.val_df = self.df[self.df.folds == self.fold].reset_index(drop=True)
    self.dff = self.train_df if self.mode == "train" else self.val_df

def __len__(self):
    return len(self.dff)

def __getitem__(self, index):

    img_name = self.dff.loc[index, "paths"]
    text = self.dff.loc[index, "corrected_prompts"]

    img = Image.open(img_name).convert("RGB")          
    pixel_values = self.processor.feature_extractor(img, return_tensors="pt").pixel_values
    labels = self.processor.tokenizer(text, 
                            padding="max_length", 
                            max_length=77,
                            truncation=True).input_ids
    labels = [label if label != self.processor.tokenizer.pad_token_id else -100 for label in labels]
    return {"input_ids":torch.tensor(labels), "pixel_values":pixel_values.squeeze()}## Trainerdefault_data_collator = DefaultDataCollator()training_args = TrainingArguments(output_dir=‘./results’,num_train_epochs=3,warmup_steps=500,weight_decay=0.01,logging_dir=‘./logs’,logging_steps=10,evaluation_strategy=‘epoch’)trainer = Trainer(model=model,args=training_args,train_dataset=train_data,eval_dataset=val_data,data_collator=default_data_collator)trainer.train()## The Error StackTracein <cell line: 22>:22                                                                            ││                                                                                                  ││   19 │   data_collator=default_data_collator                                                     ││   20 )                                                                                           ││   21                                                                                             ││ ❱ 22 trainer.train()                                                                             ││   23                                                                                             ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/trainer.py:1543 in train           ││                                                                                                  ││   1540 │   │   inner_training_loop = find_executable_batch_size(                                 ││   1541 │   │   │   self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size  ││   1542 │   │   )                                                                                 ││ ❱ 1543 │   │   return inner_training_loop(                                                       ││   1544 │   │   │   args=args,                                                                    ││   1545 │   │   │   resume_from_checkpoint=resume_from_checkpoint,                                ││   1546 │   │   │   trial=trial,                                                                  ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/trainer.py:1791 in                 ││ _inner_training_loop                                                                             ││                                                                                                  ││   1788 │   │   │   │   │   with model.no_sync():                                                 ││   1789 │   │   │   │   │   │   tr_loss_step = self.training_step(model, inputs)                  ││   1790 │   │   │   │   else:                                                                     ││ ❱ 1791 │   │   │   │   │   tr_loss_step = self.training_step(model, inputs)                      ││   1792 │   │   │   │                                                                             ││   1793 │   │   │   │   if (                                                                      ││   1794 │   │   │   │   │   args.logging_nan_inf_filter                                           ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/trainer.py:2539 in training_step   ││                                                                                                  ││   2536 │   │   │   return loss_mb.reduce_mean().detach().to(self.args.device)                    ││   2537 │   │                                                                                     ││   2538 │   │   with self.compute_loss_context_manager():                                         ││ ❱ 2539 │   │   │   loss = self.compute_loss(model, inputs)                                       ││   2540 │   │                                                                                     ││   2541 │   │   if self.args.n_gpu > 1:                                                           ││   2542 │   │   │   loss = loss.mean()  # mean() to average on multi-gpu parallel training        ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/trainer.py:2571 in compute_loss    ││                                                                                                  ││   2568 │   │   │   labels = inputs.pop(“labels”)                                                 ││   2569 │   │   else:                                                                             ││   2570 │   │   │   labels = None                                                                 ││ ❱ 2571 │   │   outputs = model(**inputs)                                                         ││   2572 │   │   # Save past state if it exists                                                    ││   2573 │   │   # TODO: this needs to be fixed and made cleaner later.                            ││   2574 │   │   if self.args.past_index >= 0:                                                     ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/modules/module.py:1110 in _call_impl   ││                                                                                                  ││   1107 │   │   # this function, and just call forward.                                           ││   1108 │   │   if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o  ││   1109 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   ││ ❱ 1110 │   │   │   return forward_call(*input, **kwargs)                                         ││   1111 │   │   # Do not call functions when jit is used                                          ││   1112 │   │   full_backward_hooks, non_full_backward_hooks =,││   1113 │   │   if self._backward_hooks or _global_backward_hooks:                                ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1125  ││ in forward                                                                                       ││                                                                                                  ││   1122 │   │   │   return_dict=return_dict,                                                      ││   1123 │   │   )                                                                                 ││   1124 │   │                                                                                     ││ ❱ 1125 │   │   text_outputs = self.text_model(                                                   ││   1126 │   │   │   input_ids=input_ids,                                                          ││   1127 │   │   │   attention_mask=attention_mask,                                                ││   1128 │   │   │   position_ids=position_ids,                                                    ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/modules/module.py:1110 in _call_impl   ││                                                                                                  ││   1107 │   │   # this function, and just call forward.                                           ││   1108 │   │   if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o  ││   1109 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   ││ ❱ 1110 │   │   │   return forward_call(*input, **kwargs)                                         ││   1111 │   │   # Do not call functions when jit is used                                          ││   1112 │   │   full_backward_hooks, non_full_backward_hooks =,││   1113 │   │   if self._backward_hooks or _global_backward_hooks:                                ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:712   ││ in forward                                                                                       ││                                                                                                  ││    709 │   │   input_shape = input_ids.size()                                                    ││    710 │   │   input_ids = input_ids.view(-1, input_shape[-1])                                   ││    711 │   │                                                                                     ││ ❱  712 │   │   hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids)   ││    713 │   │                                                                                     ││    714 │   │   bsz, seq_len = input_shape                                                        ││    715 │   │   # CLIP’s text model uses causal mask, prepare it here.                            ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/modules/module.py:1110 in _call_impl   ││                                                                                                  ││   1107 │   │   # this function, and just call forward.                                           ││   1108 │   │   if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o  ││   1109 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   ││ ❱ 1110 │   │   │   return forward_call(*input, **kwargs)                                         ││   1111 │   │   # Do not call functions when jit is used                                          ││   1112 │   │   full_backward_hooks, non_full_backward_hooks =,││   1113 │   │   if self._backward_hooks or _global_backward_hooks:                                ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:227   ││ in forward                                                                                       ││                                                                                                  ││    224 │   │   │   position_ids = self.position_ids[:, :seq_length]                              ││    225 │   │                                                                                     ││    226 │   │   if inputs_embeds is None:                                                         ││ ❱  227 │   │   │   inputs_embeds = self.token_embedding(input_ids)                               ││    228 │   │                                                                                     ││    229 │   │   position_embeddings = self.position_embedding(position_ids)                       ││    230 │   │   embeddings = inputs_embeds + position_embeddings                                  ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/modules/module.py:1110 in _call_impl   ││                                                                                                  ││   1107 │   │   # this function, and just call forward.                                           ││   1108 │   │   if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks o  ││   1109 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   ││ ❱ 1110 │   │   │   return forward_call(*input, **kwargs)                                         ││   1111 │   │   # Do not call functions when jit is used                                          ││   1112 │   │   full_backward_hooks, non_full_backward_hooks =,││   1113 │   │   if self.backward_hooks orglobal_backward_hooks:                                ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/modules/sparse.py:158 in forward       ││                                                                                                  ││   155 │   │   │   │   self.weight[self.padding_idx].fill(0)                                     ││   156 │                                                                                          ││   157 │   def forward(self, input: Tensor) → Tensor:                                            ││ ❱ 158 │   │   return F.embedding(                                                                ││   159 │   │   │   input, self.weight, self.padding_idx, self.max_norm,                           ││   160 │   │   │   self.norm_type, self.scale_grad_by_freq, self.sparse)                          ││   161                                                                                            ││                                                                                                  ││ /opt/conda/envs/blip/lib/python3.8/site-packages/torch/nn/functional.py:2183 in embedding        ││                                                                                                  ││   2180 │   │   #   torch.embedding_renorm││   2181 │   │   # remove once script supports set_grad_enabled                                    ││   2182 │   │no_grad_embedding_renorm(weight, input, max_norm, norm_type)                    ││ ❱ 2183 │   return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)        ││   2184                                                                                           ││   2185                                                                                           ││   2186 def embedding_bag(                                                                        │╰──────────────────────────────────────────────────────────────────────────────────────────────────╯IndexError: index out of range in self