A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Frontend] OpenAI Responses API supports input image (#20975)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jul 16, 2025 at 08:59 UTC
34cda778a091d4e1fd204cfde4a0f5e2b5616ac2
2 files changed
+172
-3
tests/v1/entrypoints/openai/responses/test_image.py
new
+166
@@ -0,0 +1,166 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+
6
+import openai
7
+import pytest
8
+import pytest_asyncio
9
+
10
+from tests.utils import RemoteOpenAIServer
11
+from vllm.multimodal.utils import encode_image_base64, fetch_image
12
+
13
+# Use a small vision model for testing
14
+MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"
15
+MAXIMUM_IMAGES = 2
16
+# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
17
+TEST_IMAGE_URLS = [
18
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
19
+ "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png",
20
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Venn_diagram_rgb.svg/1280px-Venn_diagram_rgb.svg.png",
21
+ "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png",
22
+]
23
+
24
+
25
+@pytest.fixture(scope="module")
26
+def default_image_server_args():
27
+ return [
28
+ "--enforce-eager",
29
+ "--max-model-len",
30
+ "6000",
31
+ "--max-num-seqs",
32
+ "128",
33
+ "--limit-mm-per-prompt",
34
+ json.dumps({"image": MAXIMUM_IMAGES}),
35
+ ]
36
+
37
+
38
+@pytest.fixture(scope="module")
39
+def image_server(default_image_server_args):
40
+ with RemoteOpenAIServer(MODEL_NAME,
41
+ default_image_server_args) as remote_server:
42
+ yield remote_server
43
+
44
+
45
+@pytest_asyncio.fixture
46
+async def client(image_server):
47
+ async with image_server.get_async_client() as async_client:
48
+ yield async_client
49
+
50
+
51
+@pytest.fixture(scope="session")
52
+def base64_encoded_image() -> dict[str, str]:
53
+ return {
54
+ image_url: encode_image_base64(fetch_image(image_url))
55
+ for image_url in TEST_IMAGE_URLS
56
+ }
57
+
58
+
59
+@pytest.mark.asyncio
60
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
61
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
62
+async def test_single_chat_session_image(client: openai.AsyncOpenAI,
63
+ model_name: str, image_url: str):
64
+ content_text = "What's in this image?"
65
+ messages = [{
66
+ "role":
67
+ "user",
68
+ "content": [
69
+ {
70
+ "type": "input_image",
71
+ "image_url": image_url,
72
+ "detail": "auto",
73
+ },
74
+ {
75
+ "type": "input_text",
76
+ "text": content_text
77
+ },
78
+ ],
79
+ }]
80
+
81
+ # test image url
82
+ response = await client.responses.create(
83
+ model=model_name,
84
+ input=messages,
85
+ )
86
+ assert len(response.output_text) > 0
87
+
88
+
89
+@pytest.mark.asyncio
90
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
91
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
92
+async def test_single_chat_session_image_base64encoded(
93
+ client: openai.AsyncOpenAI,
94
+ model_name: str,
95
+ image_url: str,
96
+ base64_encoded_image: dict[str, str],
97
+):
98
+ content_text = "What's in this image?"
99
+ messages = [{
100
+ "role":
101
+ "user",
102
+ "content": [
103
+ {
104
+ "type": "input_image",
105
+ "image_url":
106
+ f"data:image/jpeg;base64,{base64_encoded_image[image_url]}",
107
+ "detail": "auto",
108
+ },
109
+ {
110
+ "type": "input_text",
111
+ "text": content_text
112
+ },
113
+ ],
114
+ }]
115
+ # test image base64
116
+ response = await client.responses.create(
117
+ model=model_name,
118
+ input=messages,
119
+ )
120
+ assert len(response.output_text) > 0
121
+
122
+
123
+@pytest.mark.asyncio
124
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
125
+@pytest.mark.parametrize(
126
+ "image_urls",
127
+ [TEST_IMAGE_URLS[:i] for i in range(2, len(TEST_IMAGE_URLS))])
128
+async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
129
+ image_urls: list[str]):
130
+ messages = [{
131
+ "role":
132
+ "user",
133
+ "content": [
134
+ *({
135
+ "type": "input_image",
136
+ "image_url": image_url,
137
+ "detail": "auto",
138
+ } for image_url in image_urls),
139
+ {
140
+ "type": "input_text",
141
+ "text": "What's in this image?"
142
+ },
143
+ ],
144
+ }]
145
+
146
+ if len(image_urls) > MAXIMUM_IMAGES:
147
+ with pytest.raises(openai.BadRequestError): # test multi-image input
148
+ await client.responses.create(
149
+ model=model_name,
150
+ input=messages,
151
+ )
152
+ # the server should still work afterwards
153
+ response = await client.responses.create(
154
+ model=model_name,
155
+ input=[{
156
+ "role": "user",
157
+ "content": "What's the weather like in Paris today?",
158
+ }],
159
+ )
160
+ assert len(response.output_text) > 0
161
+ else:
162
+ response = await client.responses.create(
163
+ model=model_name,
164
+ input=messages,
165
+ )
166
+ assert len(response.output_text) > 0
vllm/entrypoints/chat_utils.py
+6
-3
@@ -28,6 +28,7 @@ from openai.types.chat import (ChatCompletionMessageToolCallParam,
28
ChatCompletionToolMessageParam)
29
from openai.types.chat.chat_completion_content_part_input_audio_param import (
30
InputAudio)
31
+from openai.types.responses import ResponseInputImageParam
32
from PIL import Image
33
from pydantic import BaseModel, ConfigDict, TypeAdapter
34
# yapf: enable
@@ -942,6 +943,8 @@ _ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
943
_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
944
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
945
946
+_ResponsesInputImageParser = TypeAdapter(
947
+ ResponseInputImageParam).validate_python
948
_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio, PILImage]
949
950
# Define a mapping from part types to their corresponding parsing functions.
@@ -953,6 +956,8 @@ MM_PARSER_MAP: dict[
956
lambda part: _TextParser(part).get("text", None),
957
"input_text":
958
lambda part: _TextParser(part).get("text", None),
959
+ "input_image":
960
+ lambda part: _ResponsesInputImageParser(part).get("image_url", None),
961
"image_url":
962
lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
963
"image_embeds":
@@ -1085,10 +1090,8 @@ def _parse_chat_message_content_part(
1090
"""
1091
if isinstance(part, str): # Handle plain text parts
1092
return part
1088
-
1093
# Handle structured dictionary parts
1094
part_type, content = _parse_chat_message_content_mm_part(part)
1091
-
1095
# if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1096
# content is None, log a warning and skip
1097
if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:
@@ -1109,7 +1112,7 @@ def _parse_chat_message_content_part(
1112
image_content = cast(Image.Image, content)
1113
mm_parser.parse_image_pil(image_content)
1114
modality = "image"
1112
- elif part_type == "image_url":
1115
+ elif part_type in ("image_url", "input_image"):
1116
str_content = cast(str, content)
1117
mm_parser.parse_image(str_content)
1118
modality = "image"