A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 23:00 UTC
Fix `hf_override_fn` when it modifies `model_type` (#35200)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 10, 2026 at 15:03 UTC
d88f28da05b12bc7d63ebe3dcedf445ecb274343
2 files changed
+15
-6
vllm/transformers_utils/config.py
+11
-2
@@ -161,7 +161,16 @@ class HFConfigParser(ConfigParserBase):
161
)
162
# Allow hf_overrides to override model_type before checking _CONFIG_REGISTRY
163
if (hf_overrides := kwargs.pop("hf_overrides", None)) is not None:
164
- model_type = hf_overrides.get("model_type", model_type)
164
+ if isinstance(hf_overrides, dict) and "model_type" in hf_overrides:
165
+ model_type = hf_overrides["model_type"]
166
+ elif callable(hf_overrides):
167
+ # If hf_overrides doesn't modify model_type, it will be passed straight
168
+ # through and remain unchanged by this elif block
169
+ dummy_model_type = f"dummy_{model_type}"
170
+ dummy_kwargs = dict(architectures=[""], model_type=dummy_model_type)
171
+ dummy_config = PretrainedConfig(**dummy_kwargs)
172
+ dummy_model_type = hf_overrides(dummy_config).model_type
173
+ model_type = dummy_model_type.removeprefix("dummy_")
174
175
if model_type in _CONFIG_REGISTRY:
176
config_class = _CONFIG_REGISTRY[model_type]
@@ -634,7 +643,7 @@ def get_config(
643
trust_remote_code=trust_remote_code,
644
revision=revision,
645
code_revision=code_revision,
637
- hf_overrides=hf_overrides_kw,
646
+ hf_overrides=hf_overrides_kw or hf_overrides_fn,
647
**kwargs,
648
)
649
vllm/transformers_utils/model_arch_config_convertor.py
+4
-4
@@ -79,10 +79,10 @@ class ModelArchConfigConvertorBase:
79
if getattr(self.hf_text_config, "hidden_size_per_head", None) is not None:
80
return self.hf_text_config.hidden_size_per_head
81
82
+ if (total_num_attention_heads := self.get_total_num_attention_heads()) == 0:
83
+ return 0
84
# FIXME(woosuk): This may not be true for all models.
83
- return (
84
- self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads
85
- )
85
+ return self.get_hidden_size() // total_num_attention_heads
86
87
def get_total_num_kv_heads(self) -> int:
88
attributes = [
@@ -96,7 +96,7 @@ class ModelArchConfigConvertorBase:
96
]
97
# For non-grouped-query attention models, the number of KV heads is
98
# equal to the number of attention heads.
99
- default_factory = lambda: self.hf_text_config.num_attention_heads
99
+ default_factory = self.get_total_num_attention_heads
100
return getattr_iter(
101
self.hf_text_config, attributes, default_factory=default_factory
102
)