A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Set weights_only=True when using torch.load() (#12366)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Jan 23, 2025 at 21:17 UTC
d3d6bb13fb62da3234addf6574922a4ec0513d04
4 files changed
+10
-6
vllm/assets/image.py
+1
-1
@@ -26,4 +26,4 @@ class ImageAsset:
26
"""
27
image_path = get_vllm_public_assets(filename=f"{self.name}.pt",
28
s3_prefix=VLM_IMAGES_DIR)
29
- return torch.load(image_path, map_location="cpu")
29
+ return torch.load(image_path, map_location="cpu", weights_only=True)
vllm/lora/models.py
+2
-1
@@ -273,7 +273,8 @@ class LoRAModel(AdapterModel):
273
new_embeddings_tensor_path)
274
elif os.path.isfile(new_embeddings_bin_file_path):
275
embeddings = torch.load(new_embeddings_bin_file_path,
276
- map_location=device)
276
+ map_location=device,
277
+ weights_only=True)
278
279
return cls.from_lora_tensors(
280
lora_model_id=get_lora_id()
vllm/model_executor/model_loader/weight_utils.py
+5
-3
@@ -93,7 +93,7 @@ def convert_bin_to_safetensor_file(
93
pt_filename: str,
94
sf_filename: str,
95
) -> None:
96
- loaded = torch.load(pt_filename, map_location="cpu")
96
+ loaded = torch.load(pt_filename, map_location="cpu", weights_only=True)
97
if "state_dict" in loaded:
98
loaded = loaded["state_dict"]
99
shared = _shared_pointers(loaded)
@@ -381,7 +381,9 @@ def np_cache_weights_iterator(
381
disable=not enable_tqdm,
382
bar_format=_BAR_FORMAT,
383
):
384
- state = torch.load(bin_file, map_location="cpu")
384
+ state = torch.load(bin_file,
385
+ map_location="cpu",
386
+ weights_only=True)
387
for name, param in state.items():
388
param_path = os.path.join(np_folder, name)
389
with open(param_path, "wb") as f:
@@ -447,7 +449,7 @@ def pt_weights_iterator(
449
disable=not enable_tqdm,
450
bar_format=_BAR_FORMAT,
451
):
450
- state = torch.load(bin_file, map_location="cpu")
452
+ state = torch.load(bin_file, map_location="cpu", weights_only=True)
453
yield from state.items()
454
del state
455
torch.cuda.empty_cache()
vllm/prompt_adapter/utils.py
+2
-1
@@ -89,6 +89,7 @@ def load_peft_weights(model_id: str,
89
adapters_weights = safe_load_file(filename, device=device)
90
else:
91
adapters_weights = torch.load(filename,
92
- map_location=torch.device(device))
92
+ map_location=torch.device(device),
93
+ weights_only=True)
94
95
return adapters_weights