A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[Misc] Add online audio_in_video test (#36775)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 15, 2026 at 15:14 UTC
143e4dccdfd8293c70c76f8d32a60ce23ecc23ea
5 files changed
+100
-1
requirements/test.in
+1
@@ -10,6 +10,7 @@ pytest-cov
10
11
# testing utils
12
albumentations # required for Nemotron Parse in test_common.py
13
+av # required for audio_in_video tests
14
backoff # required for phi4mm test
15
blobfile # required for kimi-vl test
16
einops # required for MPT, qwen-vl
requirements/test.txt
+2
@@ -62,6 +62,8 @@ attrs==24.2.0
62
# referencing
63
audioread==3.0.1
64
# via librosa
65
+av==16.1.0
66
+ # via -r requirements/test.in
67
backoff==2.2.1
68
# via
69
# -r requirements/test.in
tests/entrypoints/openai/test_audio_in_video.py
new
+80
@@ -0,0 +1,80 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import base64
5
+import json
6
+
7
+import openai
8
+import pytest
9
+import pytest_asyncio
10
+
11
+from ...conftest import VideoTestAssets
12
+from ...utils import RemoteOpenAIServer
13
+
14
+MODEL_NAME = "Qwen/Qwen2.5-Omni-3B"
15
+
16
+
17
+@pytest.fixture
18
+def server():
19
+ args = [
20
+ "--max-model-len",
21
+ "8192",
22
+ "--enforce-eager",
23
+ "--limit-mm-per-prompt",
24
+ json.dumps({"audio": 1, "video": 1}),
25
+ ]
26
+
27
+ with RemoteOpenAIServer(
28
+ MODEL_NAME,
29
+ args,
30
+ ) as remote_server:
31
+ yield remote_server
32
+
33
+
34
+@pytest_asyncio.fixture
35
+async def client(server):
36
+ async with server.get_async_client() as async_client:
37
+ yield async_client
38
+
39
+
40
+@pytest.mark.core_model
41
+@pytest.mark.asyncio
42
+async def test_online_audio_in_video(
43
+ client: openai.AsyncOpenAI, video_assets: VideoTestAssets
44
+):
45
+ """Test video input with `audio_in_video=True`"""
46
+
47
+ # we don't use video_urls above because they missed audio stream.
48
+ video_path = video_assets[0].video_path
49
+ with open(video_path, "rb") as f:
50
+ video_base64 = base64.b64encode(f.read()).decode("utf-8")
51
+
52
+ messages = [
53
+ {
54
+ "role": "user",
55
+ "content": [
56
+ {"type": "text", "text": "What's in this video?"},
57
+ {
58
+ "type": "video_url",
59
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
60
+ },
61
+ ],
62
+ }
63
+ ]
64
+
65
+ # multi-turn to test mm processor cache as well
66
+ for _ in range(2):
67
+ chat_completion = await client.chat.completions.create(
68
+ model=MODEL_NAME,
69
+ messages=messages,
70
+ max_tokens=16,
71
+ extra_body={
72
+ "mm_processor_kwargs": {
73
+ "use_audio_in_video": True,
74
+ }
75
+ },
76
+ )
77
+
78
+ assert len(chat_completion.choices) == 1
79
+ choice = chat_completion.choices[0]
80
+ assert choice.finish_reason == "length"
tests/multimodal/media/test_audio.py
+11
@@ -4,6 +4,7 @@ import base64
4
from pathlib import Path
5
from unittest.mock import patch
6
7
+import librosa
8
import numpy as np
9
import pytest
10
@@ -71,3 +72,13 @@ def test_audio_media_io_encode_base64(dummy_audio):
72
decoded = base64.b64decode(out)
73
assert decoded == b"dummy_wav_data"
74
mock_write.assert_called_once()
75
+
76
+
77
+def test_audio_media_io_from_video(video_assets):
78
+ audio_io = AudioMediaIO()
79
+ video_path = video_assets[0].video_path
80
+ with open(video_path, "rb") as f:
81
+ audio, sr = audio_io.load_bytes(f.read())
82
+ audio_ref, sr_ref = librosa.load(video_path, sr=None)
83
+ assert sr == sr_ref
84
+ np.testing.assert_allclose(audio_ref, audio, atol=1e-4)
vllm/entrypoints/serve/render/serving.py
+6
-1
@@ -506,6 +506,7 @@ class OpenAIServingRender:
506
(ResponsesRequest not supported here); TODO comment dropped accordingly.
507
"""
508
renderer = self.renderer
509
+ mm_config = self.model_config.multimodal_config
510
511
default_template_kwargs = merge_kwargs(
512
default_template_kwargs,
@@ -518,7 +519,11 @@ class OpenAIServingRender:
519
tok_params = request.build_tok_params(self.model_config)
520
chat_params = request.build_chat_params(
521
default_template, default_template_content_format
521
- ).with_defaults(default_template_kwargs)
522
+ ).with_defaults(
523
+ default_template_kwargs,
524
+ default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
525
+ default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
526
+ )
527
528
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
529
[messages],