A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Optimization] Use a cheaper cache key in `get_model_architecture` (#25682)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 26, 2025 at 05:54 UTC
89fa54e6f7a0789161ab73f604fdec5be6b57e58
1 file changed
+9
-2
vllm/model_executor/model_loader/utils.py
+9
-2
@@ -165,7 +165,7 @@ def device_loading_context(module: torch.nn.Module,
165
# New parameters or parameters already on target device are untouched
166
167
168
-_MODEL_ARCH_BY_HASH = dict[str, tuple[type[nn.Module], str]]()
168
+_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
169
"""Caches the outputs of `_get_model_architecture`."""
170
171
@@ -215,7 +215,14 @@ def _get_model_architecture(
215
216
def get_model_architecture(
217
model_config: ModelConfig) -> tuple[type[nn.Module], str]:
218
- key = model_config.compute_hash()
218
+ key = hash((
219
+ model_config.model,
220
+ model_config.convert_type,
221
+ model_config.runner_type,
222
+ model_config.trust_remote_code,
223
+ model_config.model_impl,
224
+ tuple(getattr(model_config.hf_config, "architectures", [])),
225
+ ))
226
if key in _MODEL_ARCH_BY_HASH:
227
return _MODEL_ARCH_BY_HASH[key]
228