A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[UX] Infer dtype for local checkpoint (#36218)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 11, 2026 at 16:50 UTC
e568cf88bc65531a95403110b186cd54dbfdc0e6
2 files changed
+8
-8
vllm/transformers_utils/config.py
+1
-1
@@ -1116,7 +1116,7 @@ def get_safetensors_params_metadata(
1116
revision: str | None = None,
1117
) -> dict[str, Any]:
1118
"""
1119
- Get the safetensors metadata for remote model repository.
1119
+ Get the safetensors parameters metadata for remote/local model repository.
1120
"""
1121
full_metadata = {}
1122
if (model_path := Path(model)).exists():
vllm/transformers_utils/model_arch_config_convertor.py
+7
-7
@@ -18,7 +18,7 @@ from vllm.config.utils import getattr_iter
18
from vllm.logger import init_logger
19
from vllm.transformers_utils.config import (
20
ConfigFormat,
21
- try_get_safetensors_metadata,
21
+ get_safetensors_params_metadata,
22
)
23
from vllm.utils.torch_utils import common_broadcastable_dtype
24
@@ -165,14 +165,14 @@ class ModelArchConfigConvertorBase:
165
# Try to read the dtype of the weights if they are in safetensors format
166
if config_dtype is None:
167
with _maybe_patch_hf_hub_constants(config_format):
168
- repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
168
+ param_mt = get_safetensors_params_metadata(model_id, revision=revision)
169
170
- if repo_mt and (files_mt := repo_mt.files_metadata):
170
+ if param_mt:
171
param_dtypes: set[torch.dtype] = {
172
- _SAFETENSORS_TO_TORCH_DTYPE[dtype_str]
173
- for file_mt in files_mt.values()
174
- for dtype_str in file_mt.parameter_count
175
- if dtype_str in _SAFETENSORS_TO_TORCH_DTYPE
172
+ _SAFETENSORS_TO_TORCH_DTYPE[dtype]
173
+ for info in param_mt.values()
174
+ if (dtype := info.get("dtype", None))
175
+ and dtype in _SAFETENSORS_TO_TORCH_DTYPE
176
}
177
178
if param_dtypes: