A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc] Fix input processing for Ultravox (#13871)
Roger Wang committed
Feb 25, 2025 at 23:56 UTC
7ca1da020f5b553ba01ace1739be9fde632f1dbb
3 files changed
+6
-15
tests/models/multimodal/processing/test_common.py
+3
-3
@@ -83,8 +83,8 @@ def _test_processing_correctness(
83
}
84
85
tokenizer_encode_kwargs = {}
86
- if model_config.hf_config.model_type in ("mllama", "whisper"):
87
- # For some encoder-decoder models, tokenizer will always add bos_token
86
+ if model_config.hf_config.model_type in ("mllama", "whisper", "ultravox"):
87
+ # For some multimodal models, tokenizer will always add bos_token
88
# at the beginning of prompt by default, causing hf_processor outputs
89
# incorrect token ids. So we need use `add_special_tokens=False` here
90
# to leave bos_token to be added by the processor.
@@ -172,7 +172,7 @@ def _test_processing_correctness(
172
"Qwen/Qwen2-VL-2B-Instruct",
173
"Qwen/Qwen2.5-VL-3B-Instruct",
174
"Qwen/Qwen2-Audio-7B-Instruct",
175
- "fixie-ai/ultravox-v0_5-llama-3_2-1b",
175
+ "fixie-ai/ultravox-v0_4",
176
"openai/whisper-large-v3",
177
])
178
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
tests/models/registry.py
+1
-1
@@ -284,7 +284,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
284
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
285
"Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
286
min_transformers_version="4.49"), # noqa: E501
287
- "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b",
287
+ "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_4",
288
trust_remote_code=True),
289
# [Encoder-decoder]
290
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
vllm/model_executor/models/ultravox.py
+2
-11
@@ -146,7 +146,8 @@ class UltravoxMultiModalProcessor(
146
) -> BatchFeature:
147
# Text-only input not supported in composite processor
148
if not mm_data or not mm_data.get("audios", []):
149
- prompt_ids = self.info.get_tokenizer().encode(prompt)
149
+ prompt_ids = self.info.get_tokenizer().encode(
150
+ prompt, add_special_tokens=False)
151
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
152
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
153
@@ -185,16 +186,6 @@ class UltravoxMultiModalProcessor(
186
)
187
return BatchFeature(combined_outputs)
188
188
- def _apply_hf_processor_tokens_only(
189
- self,
190
- prompt_tokens: list[int],
191
- ) -> list[int]:
192
- # HF processor omits bos_token_id by setting add_special_tokens=False
193
- tokenizer = self.info.get_tokenizer()
194
- assert prompt_tokens[0] == tokenizer.bos_token_id
195
-
196
- return prompt_tokens[1:]
197
-
189
def _get_mm_fields_config(
190
self,
191
hf_inputs: BatchFeature,