[Bugfix][Multimodal] Restore cached audio inputs with UUIDs (#56310)
Signed-off-by: JiataiWang <wangjiatai@proton.me> Co-authored-by: OpenAI Codex <codex@openai.com>
Jiatai Wang committed
Sep 10, 2026 at 12:20 UTC
a36dfc93cb1e4ca11425cfde48b5ed2264027aa2
4 files changed
+32
-5
tests/entrypoints/unit_tests/test_chat_utils.py
+6
-2
@@ -2384,8 +2384,10 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2384
assert conversation_with_thinking == expected_conversation
2385
2386
2387
+@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
2388
def test_parse_chat_messages_single_empty_audio_with_uuid(
2389
qwen2_audio_model_config,
2390
+ input_audio,
2391
):
2392
audio_uuid = "abcd"
2393
conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -2395,7 +2397,7 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2397
"content": [
2398
{
2399
"type": "input_audio",
2398
- "input_audio": {},
2400
+ "input_audio": input_audio,
2401
"uuid": audio_uuid,
2402
},
2403
{"type": "text", "text": "What does the audio say?"},
@@ -2418,8 +2420,10 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2420
2421
2422
@pytest.mark.asyncio
2423
+@pytest.mark.parametrize("input_audio", [None, {}, {"data": "", "format": "wav"}])
2424
async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2425
qwen2_audio_model_config,
2426
+ input_audio,
2427
):
2428
audio_uuid = "abcd"
2429
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
@@ -2429,7 +2433,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2433
"content": [
2434
{
2435
"type": "input_audio",
2432
- "input_audio": {},
2436
+ "input_audio": input_audio,
2437
"uuid": audio_uuid,
2438
},
2439
{"type": "text", "text": "What does the audio say?"},
tests/multimodal/test_parse.py
+18
@@ -6,6 +6,7 @@ import torch
6
from PIL import Image
7
8
from vllm.multimodal.parse import (
9
+ AudioProcessorItems,
10
ImageProcessorItems,
11
MultiModalDataParser,
12
VideoProcessorItems,
@@ -58,6 +59,7 @@ def test_frame_size_hwc_chw(frame):
59
@pytest.mark.parametrize(
60
"modality,processor_cls",
61
[
62
+ ("audio", AudioProcessorItems),
63
("image", ImageProcessorItems),
64
("video", VideoProcessorItems),
65
],
@@ -68,3 +70,19 @@ def test_parse_mm_data_accepts_none_cached_item(modality, processor_cls):
70
assert isinstance(items, processor_cls)
71
assert len(items) == 1
72
assert items.get(0) is None
73
+
74
+
75
+def test_cached_audio_items_preserve_positions_during_resampling():
76
+ waveform = np.arange(16, dtype=np.float32)
77
+ parser = MultiModalDataParser(
78
+ target_sr=16000, target_channels=1, audio_resample_method="scipy"
79
+ )
80
+ items = parser.parse_mm_data(
81
+ {"audio": [None, (waveform, 8000), None, (waveform, 16000)]}
82
+ )["audio"]
83
+
84
+ assert len(items) == 4
85
+ assert items.get(0) is None
86
+ assert items.get(2) is None
87
+ assert len(items.get(1)) == 32
88
+ np.testing.assert_array_equal(items.get(3), waveform)
vllm/entrypoints/chat_utils.py
+1
-1
@@ -1773,7 +1773,7 @@ def _parse_chat_message_content_mm_part(
1773
# with url as a dict of {"url": url}
1774
audio_url = audio_url.get("url", None)
1775
return "audio_url", audio_url
1776
- if part.get("input_audio") is not None:
1776
+ if "input_audio" in part:
1777
input_audio_params = _InputAudioParser(part).get("input_audio", None)
1778
return "input_audio", input_audio_params
1779
if "video_url" in part:
vllm/multimodal/parse.py
+7
-2
@@ -678,7 +678,7 @@ class MultiModalDataParser:
678
if self.is_embeddings(data):
679
return AudioEmbeddingItems(data, self.expected_hidden_size)
680
681
- data_items: list[AudioItem]
681
+ data_items: list[AudioItem | None]
682
if (
683
(is_list_of(data, float) and len(data) > 0)
684
or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 1)
@@ -690,8 +690,13 @@ class MultiModalDataParser:
690
else:
691
data_items = data # type: ignore[assignment]
692
693
- new_audios = list[np.ndarray]()
693
+ new_audios = list[np.ndarray | None]()
694
for data_item in data_items:
695
+ # Requests can omit audio samples when reusing a cached UUID.
696
+ if data_item is None:
697
+ new_audios.append(None)
698
+ continue
699
+
700
audio, orig_sr = self._get_audio_with_sr(data_item)
701
if orig_sr is None:
702
new_audio = audio