A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix typing for `safetensors_load_strategy` (#24641)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Sep 11, 2025 at 11:41 UTC
d6249d069965f88ff0042b638704f4cc66d52de4
3 files changed
+3
-4
vllm/config/load.py
+1
-1
@@ -51,7 +51,7 @@ class LoadConfig:
51
download_dir: Optional[str] = None
52
"""Directory to download and load the weights, default to the default
53
cache directory of Hugging Face."""
54
- safetensors_load_strategy: Optional[str] = "lazy"
54
+ safetensors_load_strategy: str = "lazy"
55
"""Specifies the loading strategy for safetensors weights.
56
- "lazy" (default): Weights are memory-mapped from the file. This enables
57
on-demand loading and is highly efficient for models on local storage.
vllm/engine/arg_utils.py
+1
-2
@@ -289,8 +289,7 @@ class EngineArgs:
289
trust_remote_code: bool = ModelConfig.trust_remote_code
290
allowed_local_media_path: str = ModelConfig.allowed_local_media_path
291
download_dir: Optional[str] = LoadConfig.download_dir
292
- safetensors_load_strategy: Optional[
293
- str] = LoadConfig.safetensors_load_strategy
292
+ safetensors_load_strategy: str = LoadConfig.safetensors_load_strategy
293
load_format: Union[str, LoadFormats] = LoadConfig.load_format
294
config_format: str = ModelConfig.config_format
295
dtype: ModelDType = ModelConfig.dtype
vllm/model_executor/model_loader/weight_utils.py
+1
-1
@@ -519,7 +519,7 @@ def np_cache_weights_iterator(
519
def safetensors_weights_iterator(
520
hf_weights_files: list[str],
521
use_tqdm_on_load: bool,
522
- safetensors_load_strategy: Optional[str] = "lazy",
522
+ safetensors_load_strategy: str = "lazy",
523
) -> Generator[tuple[str, torch.Tensor], None, None]:
524
"""Iterate over the weights in the model safetensor files."""
525
loading_desc = "Loading safetensors checkpoint shards"