A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Model] Ultravox: Support Llama 4 and Gemma 3 backends (#17818)
Signed-off-by: Farzad Abdolhosseini <farzad@fixie.ai> Signed-off-by: Patrick Li <patrick8289@gmail.com> Co-authored-by: Patrick Li <patrick8289@gmail.com>
Farzad Abdolhosseini committed
Jul 26, 2025 at 04:12 UTC
62965de5fe8be8e3622952a9b5cda86973cf9c51
4 files changed
+39
-24
tests/models/registry.py
+2
@@ -221,6 +221,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
221
"fp8": "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8"}), # noqa: E501
222
"LLaMAForCausalLM": _HfExamplesInfo("decapoda-research/llama-7b-hf",
223
is_available_online=False),
224
+ "Llama4ForCausalLM": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
225
+ is_available_online=False),
226
"MambaForCausalLM": _HfExamplesInfo("state-spaces/mamba-130m-hf"),
227
"Mamba2ForCausalLM": _HfExamplesInfo("mistralai/Mamba-Codestral-7B-v0.1"),
228
"FalconMambaForCausalLM": _HfExamplesInfo("tiiuae/falcon-mamba-7b-instruct"), # noqa: E501
vllm/model_executor/models/registry.py
+1
@@ -89,6 +89,7 @@ _TEXT_GENERATION_MODELS = {
89
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
90
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
91
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
92
+ "Llama4ForCausalLM": ("llama4", "Llama4ForCausalLM"), # noqa: E501
93
# For decapoda-research/llama-*
94
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
95
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
vllm/model_executor/models/ultravox.py
+23
-15
@@ -39,9 +39,7 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39
merge_multimodal_embeddings,
40
merge_multimodal_embeddings_from_map)
41
42
-_AUDIO_PLACEHOLDER_OVERRIDE = "<|reserved_special_token_0|>"
43
-_AUDIO_PLACEHOLDER_TOKEN = 128002
44
-_AUDIO_TOKENS_PER_SECOND = 6.25
42
+_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
43
_MAX_ENCODER_BATCH_SIZE = 16
44
45
@@ -80,14 +78,15 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
78
sampling_rate: Optional[int] = None,
79
**kwargs: object,
80
) -> ProcessorMixin:
81
+ config = self.ctx.model_config.hf_config
82
hf_processor = self.ctx.get_hf_processor(**kwargs)
83
84
# NOTE: Ultravox processing definition uses '<|eot_id|>' as the
85
# placeholder that will cause confusion with the actual end of turn
87
- # token, thus we override placeholder with a reserved special
88
- # token.
86
+ # token, thus we override placeholder with a reserved token.
87
hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
90
- hf_processor.audio_replacement_token_id = _AUDIO_PLACEHOLDER_TOKEN
88
+ hf_processor.audio_replacement_token_id = config.audio_token_index
89
+
90
return hf_processor
91
92
def get_feature_extractor(
@@ -274,7 +273,7 @@ class UltravoxProjector(nn.Module):
273
else:
274
self.act = get_act_fn(config.projector_act)
275
277
- dim_out = config.text_config.hidden_size
276
+ dim_out = config.text_hidden_size
277
self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
278
279
# Ultravox v0.4.1 and below use layer_norm after the second linear layer
@@ -572,9 +571,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
571
input_ids: torch.Tensor,
572
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
573
) -> torch.Tensor:
575
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
576
- if multimodal_embeddings is not None \
577
- and len(multimodal_embeddings) != 0:
574
+ # The audio token index is not included in the embedding table
575
+ # We need to remove it before embedding lookup
576
+ safe_input_ids = input_ids.clone()
577
+ safe_input_ids[safe_input_ids == self.config.audio_token_index] = 0
578
+ inputs_embeds = self.language_model.get_input_embeddings(
579
+ safe_input_ids)
580
+ if multimodal_embeddings is not None and len(
581
+ multimodal_embeddings) > 0:
582
583
# TODO(ywang96): remove this block after v0 is deprecated.
584
if not envs.VLLM_USE_V1:
@@ -585,7 +589,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
589
else:
590
inputs_embeds = merge_multimodal_embeddings(
591
input_ids, inputs_embeds, multimodal_embeddings,
588
- _AUDIO_PLACEHOLDER_TOKEN)
592
+ self.config.audio_token_index)
593
return inputs_embeds
594
595
def forward(self,
@@ -623,10 +627,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
627
multimodal_embeddings)
628
input_ids = None
629
626
- hidden_states = self.language_model.model(input_ids,
627
- positions,
628
- intermediate_tensors,
629
- inputs_embeds=inputs_embeds)
630
+ language_model = self.language_model
631
+ if hasattr(language_model, "language_model"):
632
+ language_model = language_model.language_model
633
+
634
+ hidden_states = language_model.model(input_ids,
635
+ positions,
636
+ intermediate_tensors,
637
+ inputs_embeds=inputs_embeds)
638
return hidden_states
639
640
def compute_logits(self, hidden_states: torch.Tensor,
vllm/transformers_utils/configs/ultravox.py
+13
-9
@@ -45,6 +45,7 @@ class UltravoxConfig(transformers.PretrainedConfig):
45
"""
46
47
model_type = "ultravox"
48
+ audio_token = "<|audio|>"
49
is_composition = False
50
51
def __init__(
@@ -80,29 +81,32 @@ class UltravoxConfig(transformers.PretrainedConfig):
81
# Avoid circular import
82
from vllm.transformers_utils.config import get_config
83
83
- self.text_config = get_config(text_model_id,
84
- trust_remote_code=False)
84
+ text_config_obj = get_config(text_model_id,
85
+ trust_remote_code=False)
86
else:
87
text_config = text_config or {}
87
- self.text_config = transformers.CONFIG_MAPPING[text_config.get(
88
+ text_config_obj = transformers.CONFIG_MAPPING[text_config.get(
89
"model_type", "llama")](**text_config)
90
91
+ inner_text_config = text_config_obj.get_text_config()
92
+
93
if audio_model_id is not None:
94
# Avoid circular import
95
from vllm.transformers_utils.config import get_config
96
94
- self.audio_config = get_config(audio_model_id,
95
- trust_remote_code=False)
97
+ audio_config = get_config(audio_model_id, trust_remote_code=False)
98
else:
99
audio_config = audio_config or {}
98
- self.audio_config = transformers.CONFIG_MAPPING[audio_config.get(
100
+ audio_config = transformers.CONFIG_MAPPING[audio_config.get(
101
"model_type", "whisper")](**audio_config)
102
103
+ self.text_config = text_config_obj
104
+ self.audio_config = audio_config
105
self.text_model_lora_config = text_model_lora_config or {}
106
self.audio_model_lora_config = audio_model_lora_config or {}
107
104
- self.vocab_size = self.text_config.vocab_size
105
-
106
- self.initializer_range = self.text_config.initializer_range
108
+ self.vocab_size = inner_text_config.vocab_size
109
+ self.initializer_range = inner_text_config.initializer_range
110
+ self.text_hidden_size = inner_text_config.hidden_size
111
112
super().__init__(**kwargs)