(dataset, padding=0)
return dataset
Pad the dataset. This could be optimized by defining a Dataset class and padding at the batch level, but this is simpler.
Pad the dataset. This could be optimized by defining a Dataset class and padding at the batch level, but this is simpler.
Pad the dataset . This could be optimized by defining a Dataset class and padding at the batch level but this is simpler .
def pad_dataset(dataset, padding=0):
    """ Pad the dataset. This could be optimized by defining a Dataset class and padding at the batch level, but this is simpler. """
    max_l = max(len(x) for x in dataset["input_ids"])
    for name in PADDED_INPUTS:
        dataset[name] = [x + [padding if name != "lm_labels" else -100] * (max_l - len(x)) for x in dataset[name]]
    return dataset
def pad_dataset ( dataset , padding = 0 ) : max_l = max ( len ( x ) for x in dataset [ "input_ids" ] ) for name in PADDED_INPUTS : dataset [ name ] = [ x + [ padding if name != "lm_labels" else - 100 ] * ( max_l - len ( x ) ) for x in dataset [ name ] ] return dataset