A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 14:30 UTC
[`UltraVox`] Fix output type (#37224)
Signed-off-by: vasqu <antonprogamer@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Anton Vlasjuk committed
Mar 17, 2026 at 15:51 UTC
2ff0ad9694d821bd26196cb1a0ffea80d074757e
2 files changed
+9
-6
vllm/model_executor/models/ultravox.py
+8
-5
@@ -404,12 +404,14 @@ class UltravoxTransformerProjector(nn.Module, ModuleUtilsMixin):
404
kwargs["layer_head_mask"] = None
405
406
for layer in self.layers:
407
- layer_outputs = layer(
407
+ hidden_states = layer(
408
hidden_states,
409
attention_mask=extended_attention_mask,
410
**kwargs,
411
)
412
- hidden_states = layer_outputs[0]
412
+ # BC version that allows for the old tupled output
413
+ if isinstance(hidden_states, tuple):
414
+ hidden_states = hidden_states[0]
415
416
hidden_states = self.ln_post(hidden_states)
417
hidden_states = self.linear_out(hidden_states)
@@ -509,13 +511,14 @@ class ModifiedWhisperEncoder(WhisperEncoder):
511
kwargs["layer_head_mask"] = None
512
513
for encoder_layer in self.layers:
512
- layer_outputs = encoder_layer(
514
+ hidden_states = encoder_layer(
515
hidden_states,
516
attention_mask,
517
**kwargs,
518
)
517
-
518
- hidden_states = layer_outputs[0]
519
+ # BC version that allows for the old tupled output
520
+ if isinstance(hidden_states, tuple):
521
+ hidden_states = hidden_states[0]
522
523
hidden_states = self.layer_norm(hidden_states)
524
return hidden_states
vllm/transformers_utils/configs/ultravox.py
+1
-1
@@ -43,7 +43,6 @@ class UltravoxConfig(transformers.PretrainedConfig):
43
use `False`, but v0.5 and above use `True`.
44
"""
45
46
- wrapped_model_config: transformers.PretrainedConfig
46
model_type = "ultravox"
47
audio_token = "<|audio|>"
48
is_composition = False
@@ -75,6 +74,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
74
self.num_projector_layers = num_projector_layers
75
76
# N.B. May set the wrapped_model_config below.
77
+ self.wrapped_model_config: transformers.PretrainedConfig
78
self.text_model_id = text_model_id
79
if text_model_id is None:
80
text_config = text_config or {}