A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Ultravox] Fix Gemma instantiation, support quantization via --hf-overrides (#24131)
Signed-off-by: Peter Salas <peter@fixie.ai>
Peter Salas committed
Sep 10, 2025 at 22:25 UTC
f17a6aa4ec3462ad812331259c527388eb09eb0d
3 files changed
+51
-38
vllm/config/__init__.py
+8
-4
@@ -1091,11 +1091,11 @@ class ModelConfig:
1091
1092
assert_never(runner_type)
1093
1094
- def _parse_quant_hf_config(self):
1095
- quant_cfg = getattr(self.hf_config, "quantization_config", None)
1094
+ def _parse_quant_hf_config(self, hf_config: PretrainedConfig):
1095
+ quant_cfg = getattr(hf_config, "quantization_config", None)
1096
if quant_cfg is None:
1097
# compressed-tensors uses a "compression_config" key
1098
- quant_cfg = getattr(self.hf_config, "compression_config", None)
1098
+ quant_cfg = getattr(hf_config, "compression_config", None)
1099
1100
else:
1101
# Set quant_method for ModelOpt models.
@@ -1136,7 +1136,11 @@ class ModelConfig:
1136
self.quantization)
1137
1138
# Parse quantization method from the HF model config, if available.
1139
- quant_cfg = self._parse_quant_hf_config()
1139
+ quant_cfg = self._parse_quant_hf_config(self.hf_config)
1140
+ if quant_cfg is None and (text_config := getattr(
1141
+ self.hf_config, "text_config", None)):
1142
+ # Check the text config as well for multi-modal models.
1143
+ quant_cfg = self._parse_quant_hf_config(text_config)
1144
1145
if quant_cfg is not None:
1146
# Use the community standard 'quant_method'
vllm/model_executor/models/ultravox.py
+1
-1
@@ -276,7 +276,7 @@ class UltravoxProjector(nn.Module):
276
else:
277
self.act = get_act_fn(config.projector_act)
278
279
- dim_out = config.text_hidden_size
279
+ dim_out = config.text_config.hidden_size
280
self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
281
282
# Ultravox v0.4.1 and below use layer_norm after the second linear layer
vllm/transformers_utils/configs/ultravox.py
+42
-33
@@ -20,10 +20,13 @@ class UltravoxConfig(transformers.PretrainedConfig):
20
21
Args:
22
audio_config (`Union[AutoConfig, dict]`, *optional*):
23
- Custom audio config or dict
23
+ Custom audio config or dict.
24
text_config (`Union[AutoConfig, dict]`, *optional*):
25
- The config object of the text backbone. Can be any of `LlamaConfig`
26
- or `MistralConfig`.
25
+ The config object of the text backbone.
26
+ audio_model_id (`str`, *optional*):
27
+ The model ID of the audio backbone.
28
+ text_model_id (`str`, *optional*):
29
+ The model ID of the text backbone.
30
ignore_index (`int`, *optional*, defaults to -100):
31
The ignore index for the loss function.
32
audio_token_index (`int`, *optional*, defaults to 32000):
@@ -60,15 +63,10 @@ class UltravoxConfig(transformers.PretrainedConfig):
63
stack_factor: int = 8,
64
norm_init: float = 0.4,
65
projector_act: str = "swiglu",
63
- text_model_lora_config: Optional[dict[str, Any]] = None,
64
- audio_model_lora_config: Optional[dict[str, Any]] = None,
66
projector_ln_mid: bool = False,
67
**kwargs,
68
):
69
self.ignore_index = ignore_index
69
-
70
- self.audio_model_id = audio_model_id
71
- self.text_model_id = text_model_id
70
self.audio_token_index = audio_token_index
71
72
self.hidden_size = hidden_size
@@ -77,36 +75,47 @@ class UltravoxConfig(transformers.PretrainedConfig):
75
self.projector_act = projector_act
76
self.projector_ln_mid = projector_ln_mid
77
80
- if text_model_id is not None:
81
- # Avoid circular import
82
- from vllm.transformers_utils.config import get_config
83
-
84
- text_config_obj = get_config(text_model_id,
85
- trust_remote_code=False)
86
- else:
78
+ # N.B. May set the wrapped_model_config below.
79
+ self.text_model_id = text_model_id
80
+ if text_model_id is None:
81
text_config = text_config or {}
88
- text_config_obj = transformers.CONFIG_MAPPING[text_config.get(
89
- "model_type", "llama")](**text_config)
82
+ self.wrapped_model_config = transformers.CONFIG_MAPPING[
83
+ text_config.get("model_type", "llama")](**text_config)
84
91
- inner_text_config = text_config_obj.get_text_config()
85
+ # N.B. May set the audio_config below.
86
+ self.audio_model_id = audio_model_id
87
+ if audio_model_id is None:
88
+ self.audio_model_id = None
89
+ audio_config = audio_config or {}
90
+ self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
91
+ "model_type", "whisper")](**audio_config)
92
+
93
+ super().__init__(**kwargs)
94
93
- if audio_model_id is not None:
94
- # Avoid circular import
95
+ def __setattr__(self, key, value):
96
+ # Since --hf-overrides are applied _after_ the UltravoxConfig is
97
+ # instantiated, load the configs implicitly when assigning text_model_id
98
+ # or audio_model_id. This allows:
99
+ #
100
+ # --hf-overrides.text_model_id=<quantized variant>
101
+ #
102
+ # to behave as intended.
103
+ if key == "text_model_id" and value is not None:
104
from vllm.transformers_utils.config import get_config
105
97
- audio_config = get_config(audio_model_id, trust_remote_code=False)
98
- else:
99
- audio_config = audio_config or {}
100
- audio_config = transformers.CONFIG_MAPPING[audio_config.get(
101
- "model_type", "whisper")](**audio_config)
106
+ self.wrapped_model_config = get_config(value,
107
+ trust_remote_code=False)
108
+ elif key == "audio_model_id" and value is not None:
109
+ from vllm.transformers_utils.config import get_config
110
103
- self.text_config = text_config_obj
104
- self.audio_config = audio_config
105
- self.text_model_lora_config = text_model_lora_config or {}
106
- self.audio_model_lora_config = audio_model_lora_config or {}
111
+ self.audio_config = get_config(value, trust_remote_code=False)
112
108
- self.vocab_size = inner_text_config.vocab_size
109
- self.initializer_range = inner_text_config.initializer_range
110
- self.text_hidden_size = inner_text_config.hidden_size
113
+ return super().__setattr__(key, value)
114
112
- super().__init__(**kwargs)
115
+ @property
116
+ def text_config(self) -> Optional[transformers.PretrainedConfig]:
117
+ # When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate
118
+ # the full model, but the text config is the text config of the inner
119
+ # model.
120
+ return (self.wrapped_model_config.get_text_config()
121
+ if self.wrapped_model_config else None)