0
$\begingroup$

I train a fine-tuning model with the PyTorch Trainer class:

bln_truncation = False

dataset = load_dataset("text", data_files={"train": file_path})

block_size = 512
tokenizer = AutoTokenizer.from_pretrained(model_name)

def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=bln_truncation)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)

training_args = TrainingArguments(
    output_dir="./" + model_name,
    overwrite_output_dir=True,
    num_train_epochs=num_train_epochs,
    per_device_train_batch_size=per_device_train_batch_size,
    save_steps=save_steps,
)
#     print(next(model.parameters()).device)

model = AutoModelForCausalLM.from_pretrained(model_name)
model = torch.nn.DataParallel(model)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=tokenized_datasets["train"],
)
#     print(next(model.parameters()).device)

trainer.train()

I get the warning and error:

PyTorch: setting up devices
The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).
The following columns in the training set don't have a corresponding argument in `DataParallel.forward` and have been ignored: attention_mask, input_ids, text. If attention_mask, input_ids, text are not expected by `DataParallel.forward`,  you can safely ignore this message.

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In [18], line 36
     28 trainer = Trainer(
     29     model=model,
     30     args=training_args,
     31     data_collator=data_collator,
     32     train_dataset=tokenized_datasets["train"],
     33 )
     35 # Start training
---> 36 trainer.train()

File ~/.local/lib/python3.9/site-packages/transformers/trainer.py:1317, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1312     self.model_wrapped = self.model
   1314 inner_training_loop = find_executable_batch_size(
   1315     self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size
   1316 )
-> 1317 return inner_training_loop(
   1318     args=args,
   1319     resume_from_checkpoint=resume_from_checkpoint,
   1320     trial=trial,
   1321     ignore_keys_for_eval=ignore_keys_for_eval,
   1322 )

File ~/.local/lib/python3.9/site-packages/transformers/trainer.py:1329, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1327 self._train_batch_size = batch_size
   1328 # Data loader and number of training steps
-> 1329 train_dataloader = self.get_train_dataloader()
   1331 # Setting up training control variables:
   1332 # number of training epochs: num_train_epochs
   1333 # number of training steps per epoch: num_update_steps_per_epoch
   1334 # total number of training steps to execute: max_steps
   1335 total_train_batch_size = args.train_batch_size * args.gradient_accumulation_steps * args.world_size

File ~/.local/lib/python3.9/site-packages/transformers/trainer.py:769, in Trainer.get_train_dataloader(self)
    759     return DataLoader(
    760         train_dataset,
    761         batch_size=self.args.per_device_train_batch_size,
   (...)
    764         pin_memory=self.args.dataloader_pin_memory,
    765     )
    767 train_sampler = self._get_train_sampler()
--> 769 return DataLoader(
    770     train_dataset,
    771     batch_size=self._train_batch_size,
    772     sampler=train_sampler,
    773     collate_fn=data_collator,
    774     drop_last=self.args.dataloader_drop_last,
    775     num_workers=self.args.dataloader_num_workers,
    776     pin_memory=self.args.dataloader_pin_memory,
    777     worker_init_fn=seed_worker,
    778 )

File /srv/home/seid/miniconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py:357, in DataLoader.__init__(self, dataset, batch_size, shuffle, sampler, batch_sampler, num_workers, collate_fn, pin_memory, drop_last, timeout, worker_init_fn, multiprocessing_context, generator, prefetch_factor, persistent_workers, pin_memory_device)
    353             sampler = SequentialSampler(dataset)  # type: ignore[arg-type]
    355 if batch_size is not None and batch_sampler is None:
    356     # auto_collation without custom batch_sampler
--> 357     batch_sampler = BatchSampler(sampler, batch_size, drop_last)
    359 self.batch_size = batch_size
    360 self.drop_last = drop_last

File /srv/home/seid/miniconda3/lib/python3.9/site-packages/torch/utils/data/sampler.py:232, in BatchSampler.__init__(self, sampler, batch_size, drop_last)
    226 def __init__(self, sampler: Union[Sampler[int], Iterable[int]], batch_size: int, drop_last: bool) -> None:
    227     # Since collections.abc.Iterable does not check for `__getitem__`, which
    228     # is one way for an object to be an iterable, we don't do an `isinstance`
    229     # check here.
    230     if not isinstance(batch_size, int) or isinstance(batch_size, bool) or \
    231             batch_size <= 0:
--> 232         raise ValueError("batch_size should be a positive integer value, "
    233                          "but got batch_size={}".format(batch_size))
    234     if not isinstance(drop_last, bool):
    235         raise ValueError("drop_last should be a boolean value, but got "
    236                          "drop_last={}".format(drop_last))

ValueError: batch_size should be a positive integer value, but got batch_size=11111111

I saw the warning at the beginning also in the remarks at ValueError when pre-training BERT model using Trainer API:

The following columns in the training set don't have a corresponding argument in BertForMaskedLM.forward and have been ignored: Text, Sentiment.

What can I do to get rid of the warning:

The following columns in the training set don't have a corresponding argument in DataParallel.forward and have been ignored: attention_mask, input_ids, text. If attention_mask, input_ids, text are not expected by DataParallel.forward, you can safely ignore this message.

And the error:

ValueError: batch_size should be a positive integer value, but got batch_size=11111111"?

$\endgroup$

1 Answer 1

2
$\begingroup$

I had an outdated version of transformers. I upgraded thetransformers package from:

import transformers
print(transformers.__version__)

Out:

4.19.2

To:

pip install --upgrade transformers

Out:

4.37.1

Then restart the kernel in Jupyter Notebook, or you restart your IDE.

$\endgroup$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.