A high-throughput and memory-efficient inference and serving engine for LLMs
[CI] Consolidate multimodal entrypoint tests. (#44819)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 8, 2026 at 19:48 UTC
93ee4cd47f97f547cc4d2ecd23408dfe37db57ae
24 files changed
+107
-62
.buildkite/test-amd.yaml
+29
-1
@@ -1270,6 +1270,20 @@ steps:
1270
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1271
- pytest -v -s entrypoints/speech_to_text
1272
1273
+- label: Entrypoints Integration (Multimodal)
1274
+ timeout_in_minutes: 180
1275
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
1276
+ agent_pool: mi300_1
1277
+ fast_check: true
1278
+ torch_nightly: true
1279
+ working_dir: "/vllm-workspace/tests"
1280
+ source_file_dependencies:
1281
+ - vllm/
1282
+ - tests/entrypoints/multimodal
1283
+ commands:
1284
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
1285
+ - pytest -v -s entrypoints/multimodal
1286
+
1287
- label: Entrypoints Integration (LLM) # TBD
1288
timeout_in_minutes: 180
1289
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -1327,7 +1341,7 @@ steps:
1341
- vllm/platforms/rocm.py
1342
commands:
1343
- pytest -v -s entrypoints/openai/tool_parsers
1330
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1344
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/pooling --ignore=entrypoints/multimodal --ignore=entrypoints/generate
1345
1346
- label: OpenAI API correctness # TBD
1347
timeout_in_minutes: 180
@@ -2865,6 +2879,20 @@ steps:
2879
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2880
- pytest -v -s entrypoints/speech_to_text
2881
2882
+- label: Entrypoints Integration (Multimodal)
2883
+ timeout_in_minutes: 180
2884
+ mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355]
2885
+ agent_pool: mi355_1
2886
+ fast_check: true
2887
+ torch_nightly: true
2888
+ working_dir: "/vllm-workspace/tests"
2889
+ source_file_dependencies:
2890
+ - vllm/
2891
+ - tests/entrypoints/multimodal
2892
+ commands:
2893
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
2894
+ - pytest -v -s entrypoints/multimodal
2895
+
2896
- label: Entrypoints Integration (Pooling) # TBD
2897
timeout_in_minutes: 180
2898
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
.buildkite/test_areas/entrypoints.yaml
+13
-1
@@ -11,7 +11,7 @@ steps:
11
- tests/entrypoints/
12
commands:
13
- pytest -v -s entrypoints/openai/tool_parsers
14
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
14
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/multimodal --ignore=entrypoints/generate
15
16
- label: Entrypoints Integration (LLM)
17
key: entrypoints-integration-llm
@@ -120,6 +120,18 @@ steps:
120
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
121
- pytest -v -s entrypoints/speech_to_text
122
123
+- label: Entrypoints Integration (Multimodal)
124
+ device: h200_35gb
125
+ key: entrypoints-integration-multimodal
126
+ timeout_in_minutes: 50
127
+ working_dir: "/vllm-workspace/tests"
128
+ source_file_dependencies:
129
+ - vllm/
130
+ - tests/entrypoints/multimodal
131
+ commands:
132
+ - export VLLM_WORKER_MULTIPROC_METHOD=spawn
133
+ - pytest -v -s entrypoints/multimodal
134
+
135
- label: Entrypoints Integration (Pooling)
136
key: entrypoints-integration-pooling
137
timeout_in_minutes: 50
tests/entrypoints/llm/test_chat.py
-42
@@ -4,7 +4,6 @@ import weakref
4
5
import pytest
6
7
-from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
7
from vllm import LLM
8
from vllm.distributed import cleanup_dist_env_and_memory
9
from vllm.sampling_params import SamplingParams
@@ -76,47 +75,6 @@ def test_multi_chat(text_llm):
75
assert len(outputs) == 2
76
77
79
-@pytest.fixture(scope="function")
80
-def vision_llm():
81
- # pytest caches the fixture so we use weakref.proxy to
82
- # enable garbage collection
83
- llm = LLM(
84
- model="microsoft/Phi-3.5-vision-instruct",
85
- max_model_len=4096,
86
- max_num_seqs=5,
87
- enforce_eager=True,
88
- trust_remote_code=True,
89
- limit_mm_per_prompt={"image": 2},
90
- seed=0,
91
- )
92
-
93
- yield weakref.proxy(llm)
94
-
95
- del llm
96
-
97
- cleanup_dist_env_and_memory()
98
-
99
-
100
-@pytest.mark.parametrize(
101
- "image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
102
-)
103
-def test_chat_multi_image(vision_llm, image_urls: list[str]):
104
- messages = [
105
- {
106
- "role": "user",
107
- "content": [
108
- *(
109
- {"type": "image_url", "image_url": {"url": image_url}}
110
- for image_url in image_urls
111
- ),
112
- {"type": "text", "text": "What's in this image?"},
113
- ],
114
- }
115
- ]
116
- outputs = vision_llm.chat(messages)
117
- assert len(outputs) >= 0
118
-
119
-
78
def test_llm_chat_tokenization_no_double_bos(text_llm):
79
"""
80
LLM.chat() should not add special tokens when using chat templates.
tests/entrypoints/multimodal/__init__.py
tests/entrypoints/multimodal/conftest.py
new
+10
@@ -0,0 +1,10 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
5
+TEST_IMAGE_ASSETS = [
6
+ "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
7
+ "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
8
+ "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
9
+ "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
10
+]
tests/entrypoints/multimodal/llm/__init__.py
tests/entrypoints/multimodal/llm/test_chat.py
new
+50
@@ -0,0 +1,50 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import weakref
4
+
5
+import pytest
6
+
7
+from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
8
+from vllm import LLM
9
+from vllm.distributed import cleanup_dist_env_and_memory
10
+
11
+
12
+@pytest.fixture(scope="function")
13
+def vision_llm():
14
+ # pytest caches the fixture so we use weakref.proxy to
15
+ # enable garbage collection
16
+ llm = LLM(
17
+ model="microsoft/Phi-3.5-vision-instruct",
18
+ max_model_len=4096,
19
+ max_num_seqs=5,
20
+ enforce_eager=True,
21
+ trust_remote_code=True,
22
+ limit_mm_per_prompt={"image": 2},
23
+ seed=0,
24
+ )
25
+
26
+ yield weakref.proxy(llm)
27
+
28
+ del llm
29
+
30
+ cleanup_dist_env_and_memory()
31
+
32
+
33
+@pytest.mark.parametrize(
34
+ "image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
35
+)
36
+def test_chat_multi_image(vision_llm, image_urls: list[str]):
37
+ messages = [
38
+ {
39
+ "role": "user",
40
+ "content": [
41
+ *(
42
+ {"type": "image_url", "image_url": {"url": image_url}}
43
+ for image_url in image_urls
44
+ ),
45
+ {"type": "text", "text": "What's in this image?"},
46
+ ],
47
+ }
48
+ ]
49
+ outputs = vision_llm.chat(messages)
50
+ assert len(outputs) >= 0
tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py
renamed
+1
-1
@@ -15,7 +15,7 @@ import logging
15
import pytest
16
import regex as re
17
18
-from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
18
+from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
19
from vllm import LLM, SamplingParams
20
from vllm.renderers.params import ChatParams
21
from vllm.v1.metrics import loggers as stat_loggers
tests/entrypoints/multimodal/llm/test_mm_cache_stats.py
renamed
+1
-1
@@ -6,7 +6,7 @@ import logging
6
import pytest
7
import regex as re
8
9
-from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
9
+from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
10
from vllm import LLM
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
from vllm.v1.metrics import loggers as stat_loggers
tests/entrypoints/multimodal/llm/test_mm_embeds_only.py
renamed
tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py
renamed
tests/entrypoints/multimodal/openai/__init__.py
tests/entrypoints/multimodal/openai/chat_completion/__init__.py
tests/entrypoints/multimodal/openai/chat_completion/test_audio.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_image_embeds.py
renamed
+1
-1
@@ -52,7 +52,7 @@ async def client_with_image_embeds(server_with_image_embeds):
52
@pytest.mark.asyncio
53
@pytest.mark.parametrize("model_name", [MODEL_NAME])
54
@pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32])
55
-async def test_completions_with_image_embeds(
55
+async def test_chat_completions_with_image_embeds(
56
client_with_image_embeds: openai.AsyncOpenAI,
57
model_name: str,
58
image_assets: ImageTestAssets,
tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_audio_embeds.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_image_embeds.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_default_mm_loras.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_video.py
renamed
tests/entrypoints/multimodal/openai/chat_completion/test_vision.py
renamed
+1
-8
@@ -8,6 +8,7 @@ import pytest
8
import pytest_asyncio
9
from transformers import AutoProcessor
10
11
+from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
12
from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
13
from vllm.multimodal.media import MediaWithBytes
14
from vllm.multimodal.utils import encode_image_url, fetch_image
@@ -16,14 +17,6 @@ from vllm.platforms import current_platform
17
MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
18
MAXIMUM_IMAGES = 2
19
19
-# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
20
-TEST_IMAGE_ASSETS = [
21
- "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
22
- "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
23
- "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
24
- "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
25
-]
26
-
20
# Required terms for beam search validation
21
# Each entry is a list of term groups - ALL groups must match
22
# Each group is a list of alternatives - at least ONE term in the group must appear
tests/entrypoints/multimodal/openai/chat_completion/test_vision_embeds.py
renamed
tests/entrypoints/multimodal/openai/responses/__init__.py
tests/entrypoints/multimodal/openai/responses/test_image.py
renamed
+1
-7
@@ -7,19 +7,13 @@ import openai
7
import pytest
8
import pytest_asyncio
9
10
+from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
11
from tests.utils import RemoteOpenAIServer
12
from vllm.multimodal.utils import encode_image_url
13
14
# Use a small vision model for testing
15
MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"
16
MAXIMUM_IMAGES = 2
16
-# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
17
-TEST_IMAGE_ASSETS = [
18
- "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
19
- "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png",
20
- "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
21
- "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
22
-]
17
18
19
@pytest.fixture(scope="module")