Not able to scale Trainer code to single node multi GPU
I am using the code provided in thisblog. Here is the link to google colab notebookhereThe notebook runs perfectly fine in a machine with single GPU. However, when I run it on machine with Mutiple GPUs (n=4, Nvidia Tesla T4), I am getting the following error (at the end)To debug, I setuse_cpu=Trueand the training loop runs ok as expected. So, I am pretty sure it is about multi-GPU.Trainer code (full code in the colab notebook)from transformers import Seq2SeqTrainingArguments

training_args = Seq2SeqTrainingArguments(
    output_dir="./speecht5_tts_voxpopuli_nl",  # change to a repo name of your choice
    per_device_train_batch_size=16,
    gradient_accumulation_steps=2,
    learning_rate=1e-5,
    warmup_steps=500,
    max_steps=4000,
    gradient_checkpointing=True,
    fp16=True,
    evaluation_strategy="steps",
    per_device_eval_batch_size=8,
    save_steps=1000,
    eval_steps=1000,
    logging_steps=25,
    report_to=["tensorboard"],
    load_best_model_at_end=True,
    greater_is_better=False,
    label_names=["labels"],
    push_to_hub=True,
)

from transformers import Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    args=training_args,
    model=model,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
    data_collator=data_collator,
    tokenizer=processor.tokenizer,
)

trainer.train()/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
  warnings.warn('Was asked to gather along dimension 0, but all '
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File <timed eval>:1

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1575, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1573         REDACTED_TOKEN_utils.enable_progress_bars()
   1574 else:
-> 1575     return inner_training_loop(
   1576         args=args,
   1577         resume_from_checkpoint=resume_from_checkpoint,
   1578         trial=trial,
   1579         ignore_keys_for_eval=ignore_keys_for_eval,
   1580     )

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1875, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1872     self.control = self.callback_handler.on_step_begin(args, self.state, self.control)
   1874 with self.accelerator.accumulate(model):
-> 1875     tr_loss_step = self.training_step(model, inputs)
   1877 if (
   1878     args.logging_nan_inf_filter
   1879     and not is_torch_tpu_available()
   1880     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
   1881 ):
   1882     # if loss is nan or inf simply add the average of previous logged losses
   1883     tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2740, in Trainer.training_step(self, model, inputs)
   2737     return loss_mb.reduce_mean().detach().to(self.args.device)
   2739 with self.compute_loss_context_manager():
-> 2740     loss = self.compute_loss(model, inputs)
   2742 if self.args.n_gpu > 1:
   2743     loss = loss.mean()  # mean() to average on multi-gpu parallel training

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2765, in Trainer.compute_loss(self, model, inputs, return_outputs)
   2763 else:
   2764     labels = None
-> 2765 outputs = model(**inputs)
   2766 # Save past state if it exists
   2767 # TODO: this needs to be fixed and made cleaner later.
   2768 if self.args.past_index >= 0:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py:172, in DataParallel.forward(self, *inputs, **kwargs)
    170 replicas = self.replicate(self.module, self.device_ids[:len(inputs)])
    171 outputs = self.parallel_apply(replicas, inputs, kwargs)
--> 172 return self.gather(outputs, self.output_device)

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py:184, in DataParallel.gather(self, outputs, output_device)
    183 def gather(self, outputs, output_device):
--> 184     return gather(outputs, output_device, dim=self.dim)

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:86, in gather(outputs, target_device, dim)
     83 # Recursive function calls like this create reference cycles.
     84 # Setting the function to None clears the refcycle.
     85 try:
---> 86     res = gather_map(outputs)
     87 finally:
     88     gather_map = None

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:77, in gather.<locals>.gather_map(outputs)
     75     if not all(len(out) == len(d) for d in outputs):
     76         raise ValueError('All dicts must have the same number of keys')
---> 77     return type(out)((k, gather_map([d[k] for d in outputs]))
     78                      for k in out)
     79 if _is_namedtuple(out):
     80     return type(out)._make(map(gather_map, zip(*outputs)))

File <string>:12, in __init__(self, loss, spectrogram, past_key_values, decoder_hidden_states, decoder_attentions, cross_attentions, encoder_last_hidden_state, encoder_hidden_states, encoder_attentions)

File /opt/conda/lib/python3.10/site-packages/transformers/utils/generic.py:343, in ModelOutput.__post_init__(self)
    340 # if we provided an iterator as first field and the iterator is a (key, value) iterator
    341 # set the associated fields
    342 if first_field_iterator:
--> 343     for idx, element in enumerate(iterator):
    344         if (
    345             not isinstance(element, (list, tuple))
    346             or not len(element) == 2
    347             or not isinstance(element[0], str)
    348         ):
    349             if idx == 0:
    350                 # If we do not have an iterator of key/values, set it as attribute

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:77, in <genexpr>(.0)
     75     if not all(len(out) == len(d) for d in outputs):
     76         raise ValueError('All dicts must have the same number of keys')
---> 77     return type(out)((k, gather_map([d[k] for d in outputs]))
     78                      for k in out)
     79 if _is_namedtuple(out):
     80     return type(out)._make(map(gather_map, zip(*outputs)))

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:81, in gather.<locals>.gather_map(outputs)
     79 if _is_namedtuple(out):
     80     return type(out)._make(map(gather_map, zip(*outputs)))
---> 81 return type(out)(map(gather_map, zip(*outputs)))

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/scatter_gather.py:71, in gather.<locals>.gather_map(outputs)
     69 out = outputs[0]
     70 if isinstance(out, torch.Tensor):
---> 71     return Gather.apply(target_device, dim, *outputs)
     72 if out is None:
     73     return None

File /opt/conda/lib/python3.10/site-packages/torch/autograd/function.py:506, in Function.apply(cls, *args, **kwargs)
    503 if not torch._C._are_functorch_transforms_active():
    504     # See NOTE: [functorch vjp and autograd interaction]
    505     args = _functorch.utils.unwrap_dead_wrappers(args)
--> 506     return super().apply(*args, **kwargs)  # type: ignore[misc]
    508 if cls.setup_context == _SingleLevelFunction.setup_context:
    509     raise RuntimeError(
    510         'In order to use an autograd.Function with functorch transforms '
    511         '(vmap, grad, jvp, jacrev, ...), it must override the setup_context '
    512         'staticmethod. For more details, please see '
    513         'https://pytorch.org/docs/master/notes/extending.func.html')

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:56, in Gather.forward(ctx, target_device, dim, *inputs)
     54 @staticmethod
     55 def forward(ctx, target_device, dim, *inputs):
---> 56     assert all(i.device.type != 'cpu' for i in inputs), (
     57         'Gather function not implemented for CPU tensors'
     58     )
     59     if (target_device == 'cpu'):
     60         ctx.target_device = 'cpu'

File /opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:56, in <genexpr>(.0)
     54 @staticmethod
     55 def forward(ctx, target_device, dim, *inputs):
---> 56     assert all(i.device.type != 'cpu' for i in inputs), (
     57         'Gather function not implemented for CPU tensors'
     58     )
     59     if (target_device == 'cpu'):
     60         ctx.target_device = 'cpu'

AttributeError: 'NoneType' object has no attribute 'device'