Cannot resume trainer from checkpoint
I am trying to resume a training session from a checkpoint. I load the original model and then I call the train(“path/to/checkpoint”) method with a path to the checkpoint. However this refuses to resume from checkpoint. The model was trained with FSDP. could this be affecting  nature of the checkpoint saves? This is the error I am getting below.Loading model from output/checkpoint-18000.Traceback (most recent call last):File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 246, intrain()File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 239, in traintrainer.train(“output/checkpoint-18000”)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 1617, in trainself._load_from_checkpoint(resume_from_checkpoint)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 2120, in _load_from_checkpointload_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_utils.py”, line 385, in load_sharded_checkpointstate_dict = torch.load(os.path.join(folder, shard_file), map_location=“cpu”)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 809, in loadreturn _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 1172, in _loadresult = unpickler.load()File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 169, in _rebuild_tensor_v2tensor = _rebuild_tensor(storage, storage_offset, size, stride)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 148, inrebuild_tensorreturn t.set(storage._untyped_storage, storage_offset, size, stride)RuntimeError: Trying to resize storage that is not resizableLoading model from output/checkpoint-18000.Loading model from output/checkpoint-18000.Loading model from output/checkpoint-18000.Traceback (most recent call last):File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 246, intrain()File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 239, in traintrainer.train(“output/checkpoint-18000”)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 1617, in trainself._load_from_checkpoint(resume_from_checkpoint)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 2120, in _load_from_checkpointload_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_utils.py”, line 385, in load_sharded_checkpointstate_dict = torch.load(os.path.join(folder, shard_file), map_location=“cpu”)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 809, in loadreturn _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 1172, in _loadresult = unpickler.load()File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 169, in _rebuild_tensor_v2tensor = _rebuild_tensor(storage, storage_offset, size, stride)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 148, inrebuild_tensorreturn t.set(storage._untyped_storage, storage_offset, size, stride)RuntimeError: Trying to resize storage that is not resizableTraceback (most recent call last):File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 246, intrain()File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 239, in traintrainer.train(“output/checkpoint-18000”)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 1617, in trainself._load_from_checkpoint(resume_from_checkpoint)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 2120, in _load_from_checkpointload_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_utils.py”, line 385, in load_sharded_checkpointstate_dict = torch.load(os.path.join(folder, shard_file), map_location=“cpu”)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 809, in loadreturn _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 1172, in _loadresult = unpickler.load()File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 169, in _rebuild_tensor_v2tensor = _rebuild_tensor(storage, storage_offset, size, stride)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 148, inrebuild_tensorreturn t.set(storage._untyped_storage, storage_offset, size, stride)RuntimeError: Trying to resize storage that is not resizableTraceback (most recent call last):File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 246, intrain()File “/home/ubuntu/alpaca/stanford_alpaca/train.py”, line 239, in traintrainer.train(“output/checkpoint-18000”)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 1617, in trainself._load_from_checkpoint(resume_from_checkpoint)File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/trainer.py”, line 2120, in _load_from_checkpointload_result = load_sharded_checkpoint(model, resume_from_checkpoint, strict=is_sagemaker_mp_enabled())File “/home/ubuntu/.local/lib/python3.10/site-packages/transformers/modeling_utils.py”, line 385, in load_sharded_checkpointstate_dict = torch.load(os.path.join(folder, shard_file), map_location=“cpu”)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 809, in loadreturn _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/serialization.py”, line 1172, in _loadresult = unpickler.load()File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 169, in _rebuild_tensor_v2tensor = _rebuild_tensor(storage, storage_offset, size, stride)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/_utils.py”, line 148, inrebuild_tensorreturn t.set(storage._untyped_storage, storage_offset, size, stride)RuntimeError: Trying to resize storage that is not resizableWARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123046 closing signal SIGTERMWARNING:torch.distributed.elastic.multiprocessing.api:Sending process 123048 closing signal SIGTERMERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 1 (pid: 123047) of binary: /usr/local/bin/python3.10Traceback (most recent call last):File “/home/ubuntu/.local/bin/torchrun”, line 8, insys.exit(main())File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/init.py”, line 346, in wrapperreturn f(*args, **kwargs)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/distributed/run.py”, line 794, in mainrun(args)File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/distributed/run.py”, line 785, in runelastic_launch(File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py”, line 134, incallreturn launch_agent(self._config, self._entrypoint, list(args))File “/home/ubuntu/.local/lib/python3.10/site-packages/torch/distributed/launcher/api.py”, line 250, in launch_agentraise ChildFailedError(torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
2023-04-09T05:28:45Z
Same error. Any solutions?
2023-05-05T09:20:46Z
I’m having the same errors when loading a checkpoint using torch.load(checkpoint)