A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend] Add OpenAI Vision API Support (#5237)
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Roger Wang committed
Jun 7, 2024 at 11:23 UTC
7a9cb294ae317b28a60165b34c8398c762869a74
9 files changed
+653
-19
docs/source/models/vlm.rst
+67
-1
@@ -3,7 +3,7 @@
3
Using VLMs
4
==========
5
6
-This document shows you how to run and serve Vision Language Models (VLMs) using vLLM.
6
+vLLM provides experimental support for Vision Language Models (VLMs). This document shows you how to run and serve these models using vLLM.
7
8
Engine Arguments
9
----------------
@@ -54,3 +54,69 @@ For now, we only support a single image per text prompt. To pass an image to the
54
print(generated_text)
55
56
A code example can be found in `examples/llava_example.py <https://github.com/vllm-project/vllm/blob/main/examples/llava_example.py>`_.
57
+
58
+Online OpenAI Vision API Compatible Inference
59
+----------------------------------------------
60
+
61
+You can serve vision language models with vLLM's HTTP server that is compatible with `OpenAI Vision API <https://platform.openai.com/docs/guides/vision>`_.
62
+
63
+.. note::
64
+ Currently, vLLM supports only **single** ``image_url`` input per ``messages``. Support for multi-image inputs will be
65
+ added in the future.
66
+
67
+Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with vLLM API server.
68
+
69
+.. important::
70
+ Since OpenAI Vision API is based on `Chat <https://platform.openai.com/docs/api-reference/chat>`_ API, a chat template
71
+ is **required** to launch the API server if the model's tokenizer does not come with one. In this example, we use the
72
+ HuggingFace Llava chat template that you can find in the example folder `here <https://github.com/vllm-project/vllm/blob/main/examples/template_llava.jinja>`_.
73
+
74
+.. code-block:: bash
75
+
76
+ python -m vllm.entrypoints.openai.api_server \
77
+ --model llava-hf/llava-1.5-7b-hf \
78
+ --image-input-type pixel_values \
79
+ --image-token-id 32000 \
80
+ --image-input-shape 1,3,336,336 \
81
+ --image-feature-size 576 \
82
+ --chat-template template_llava.jinja
83
+
84
+To consume the server, you can use the OpenAI client like in the example below:
85
+
86
+.. code-block:: python
87
+
88
+ from openai import OpenAI
89
+ openai_api_key = "EMPTY"
90
+ openai_api_base = "http://localhost:8000/v1"
91
+ client = OpenAI(
92
+ api_key=openai_api_key,
93
+ base_url=openai_api_base,
94
+ )
95
+ chat_response = client.chat.completions.create(
96
+ model="llava-hf/llava-1.5-7b-hf",
97
+ messages=[{
98
+ "role": "user",
99
+ "content": [
100
+ {"type": "text", "text": "What's in this image?"},
101
+ {
102
+ "type": "image_url",
103
+ "image_url": {
104
+ "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
105
+ },
106
+ },
107
+ ],
108
+ }],
109
+ )
110
+ print("Chat response:", chat_response)
111
+
112
+.. note::
113
+
114
+ By default, the timeout for fetching images through http url is ``5`` seconds. You can override this by setting the environment variable:
115
+
116
+ .. code-block:: shell
117
+
118
+ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
119
+
120
+.. note::
121
+ The prompt formatting with the image token ``<image>`` is not needed when serving VLMs with the API server since the prompt will be
122
+ processed automatically by the server.
docs/source/serving/openai_compatible_server.md
+3
-1
@@ -30,6 +30,8 @@ Please see the [OpenAI API Reference](https://platform.openai.com/docs/api-refer
30
- Chat: `tools`, and `tool_choice`.
31
- Completions: `suffix`.
32
33
+vLLM also provides experimental support for OpenAI Vision API compatible inference. See more details in [Using VLMs](../models/vlm.rst).
34
+
35
## Extra Parameters
36
vLLM supports a set of parameters that are not part of the OpenAI API.
37
In order to use them, you can pass them as extra parameters in the OpenAI client.
@@ -120,4 +122,4 @@ It is the callers responsibility to prompt the model with the tool information,
122
123
vLLM will use guided decoding to ensure the response matches the tool parameter object defined by the JSON schema in the `tools` parameter.
124
123
-Please refer to the OpenAI API reference documentation for more information.
\ No newline at end of file
125
+Please refer to the OpenAI API reference documentation for more information.
examples/template_llava.jinja
new
+23
@@ -0,0 +1,23 @@
1
+{%- if messages[0]['role'] == 'system' -%}
2
+ {%- set system_message = messages[0]['content'] -%}
3
+ {%- set messages = messages[1:] -%}
4
+{%- else -%}
5
+ {% set system_message = '' -%}
6
+{%- endif -%}
7
+
8
+{{ bos_token + system_message }}
9
+{%- for message in messages -%}
10
+ {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
11
+ {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
12
+ {%- endif -%}
13
+
14
+ {%- if message['role'] == 'user' -%}
15
+ {{ 'USER: ' + message['content'] + '\n' }}
16
+ {%- elif message['role'] == 'assistant' -%}
17
+ {{ 'ASSISTANT: ' + message['content'] + eos_token + '\n' }}
18
+ {%- endif -%}
19
+{%- endfor -%}
20
+
21
+{%- if add_generation_prompt -%}
22
+ {{ 'ASSISTANT:' }}
23
+{% endif %}
tests/entrypoints/test_openai_vision.py
new
+286
@@ -0,0 +1,286 @@
1
+from pathlib import Path
2
+from typing import Dict
3
+
4
+import openai
5
+import pytest
6
+import pytest_asyncio
7
+import ray
8
+
9
+from vllm.multimodal.utils import ImageFetchAiohttp, encode_image_base64
10
+
11
+from ..utils import ServerRunner
12
+
13
+MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
14
+LLAVA_CHAT_TEMPLATE = (Path(__file__).parent.parent.parent /
15
+ "examples/template_llava.jinja")
16
+assert LLAVA_CHAT_TEMPLATE.exists()
17
+# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
18
+TEST_IMAGE_URLS = [
19
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
20
+ "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png",
21
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Venn_diagram_rgb.svg/1280px-Venn_diagram_rgb.svg.png",
22
+ "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png",
23
+]
24
+
25
+pytestmark = pytest.mark.openai
26
+
27
+
28
+@pytest.fixture(scope="module")
29
+def server():
30
+ ray.init()
31
+ server_runner = ServerRunner.remote([
32
+ "--model",
33
+ MODEL_NAME,
34
+ "--dtype",
35
+ "bfloat16",
36
+ "--max-model-len",
37
+ "4096",
38
+ "--enforce-eager",
39
+ "--image-input-type",
40
+ "pixel_values",
41
+ "--image-token-id",
42
+ "32000",
43
+ "--image-input-shape",
44
+ "1,3,336,336",
45
+ "--image-feature-size",
46
+ "576",
47
+ "--chat-template",
48
+ str(LLAVA_CHAT_TEMPLATE),
49
+ ])
50
+ ray.get(server_runner.ready.remote())
51
+ yield server_runner
52
+ ray.shutdown()
53
+
54
+
55
+@pytest.fixture(scope="session")
56
+def client():
57
+ client = openai.AsyncOpenAI(
58
+ base_url="http://localhost:8000/v1",
59
+ api_key="token-abc123",
60
+ )
61
+ yield client
62
+
63
+
64
+@pytest_asyncio.fixture(scope="session")
65
+async def base64_encoded_image() -> Dict[str, str]:
66
+ return {
67
+ image_url:
68
+ encode_image_base64(await ImageFetchAiohttp.fetch_image(image_url))
69
+ for image_url in TEST_IMAGE_URLS
70
+ }
71
+
72
+
73
+@pytest.mark.asyncio
74
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
75
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
76
+async def test_single_chat_session_image(server, client: openai.AsyncOpenAI,
77
+ model_name: str, image_url: str):
78
+ messages = [{
79
+ "role":
80
+ "user",
81
+ "content": [
82
+ {
83
+ "type": "image_url",
84
+ "image_url": {
85
+ "url": image_url
86
+ }
87
+ },
88
+ {
89
+ "type": "text",
90
+ "text": "What's in this image?"
91
+ },
92
+ ],
93
+ }]
94
+
95
+ # test single completion
96
+ chat_completion = await client.chat.completions.create(model=model_name,
97
+ messages=messages,
98
+ max_tokens=10,
99
+ logprobs=True,
100
+ top_logprobs=5)
101
+ assert len(chat_completion.choices) == 1
102
+
103
+ choice = chat_completion.choices[0]
104
+ assert choice.finish_reason == "length"
105
+ assert chat_completion.usage == openai.types.CompletionUsage(
106
+ completion_tokens=10, prompt_tokens=596, total_tokens=606)
107
+
108
+ message = choice.message
109
+ message = chat_completion.choices[0].message
110
+ assert message.content is not None and len(message.content) >= 10
111
+ assert message.role == "assistant"
112
+ messages.append({"role": "assistant", "content": message.content})
113
+
114
+ # test multi-turn dialogue
115
+ messages.append({"role": "user", "content": "express your result in json"})
116
+ chat_completion = await client.chat.completions.create(
117
+ model=model_name,
118
+ messages=messages,
119
+ max_tokens=10,
120
+ )
121
+ message = chat_completion.choices[0].message
122
+ assert message.content is not None and len(message.content) >= 0
123
+
124
+
125
+@pytest.mark.asyncio
126
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
127
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
128
+async def test_single_chat_session_image_base64encoded(
129
+ server, client: openai.AsyncOpenAI, model_name: str, image_url: str,
130
+ base64_encoded_image: Dict[str, str]):
131
+
132
+ messages = [{
133
+ "role":
134
+ "user",
135
+ "content": [
136
+ {
137
+ "type": "image_url",
138
+ "image_url": {
139
+ "url":
140
+ f"data:image/jpeg;base64,{base64_encoded_image[image_url]}"
141
+ }
142
+ },
143
+ {
144
+ "type": "text",
145
+ "text": "What's in this image?"
146
+ },
147
+ ],
148
+ }]
149
+
150
+ # test single completion
151
+ chat_completion = await client.chat.completions.create(model=model_name,
152
+ messages=messages,
153
+ max_tokens=10,
154
+ logprobs=True,
155
+ top_logprobs=5)
156
+ assert len(chat_completion.choices) == 1
157
+
158
+ choice = chat_completion.choices[0]
159
+ assert choice.finish_reason == "length"
160
+ assert chat_completion.usage == openai.types.CompletionUsage(
161
+ completion_tokens=10, prompt_tokens=596, total_tokens=606)
162
+
163
+ message = choice.message
164
+ message = chat_completion.choices[0].message
165
+ assert message.content is not None and len(message.content) >= 10
166
+ assert message.role == "assistant"
167
+ messages.append({"role": "assistant", "content": message.content})
168
+
169
+ # test multi-turn dialogue
170
+ messages.append({"role": "user", "content": "express your result in json"})
171
+ chat_completion = await client.chat.completions.create(
172
+ model=model_name,
173
+ messages=messages,
174
+ max_tokens=10,
175
+ )
176
+ message = chat_completion.choices[0].message
177
+ assert message.content is not None and len(message.content) >= 0
178
+
179
+
180
+@pytest.mark.asyncio
181
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
182
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
183
+async def test_chat_streaming_image(server, client: openai.AsyncOpenAI,
184
+ model_name: str, image_url: str):
185
+ messages = [{
186
+ "role":
187
+ "user",
188
+ "content": [
189
+ {
190
+ "type": "image_url",
191
+ "image_url": {
192
+ "url": image_url
193
+ }
194
+ },
195
+ {
196
+ "type": "text",
197
+ "text": "What's in this image?"
198
+ },
199
+ ],
200
+ }]
201
+
202
+ # test single completion
203
+ chat_completion = await client.chat.completions.create(
204
+ model=model_name,
205
+ messages=messages,
206
+ max_tokens=10,
207
+ temperature=0.0,
208
+ )
209
+ output = chat_completion.choices[0].message.content
210
+ stop_reason = chat_completion.choices[0].finish_reason
211
+
212
+ # test streaming
213
+ stream = await client.chat.completions.create(
214
+ model=model_name,
215
+ messages=messages,
216
+ max_tokens=10,
217
+ temperature=0.0,
218
+ stream=True,
219
+ )
220
+ chunks = []
221
+ finish_reason_count = 0
222
+ async for chunk in stream:
223
+ delta = chunk.choices[0].delta
224
+ if delta.role:
225
+ assert delta.role == "assistant"
226
+ if delta.content:
227
+ chunks.append(delta.content)
228
+ if chunk.choices[0].finish_reason is not None:
229
+ finish_reason_count += 1
230
+ # finish reason should only return in last block
231
+ assert finish_reason_count == 1
232
+ assert chunk.choices[0].finish_reason == stop_reason
233
+ assert delta.content
234
+ assert "".join(chunks) == output
235
+
236
+
237
+@pytest.mark.asyncio
238
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
239
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
240
+async def test_multi_image_input(server, client: openai.AsyncOpenAI,
241
+ model_name: str, image_url: str):
242
+
243
+ messages = [{
244
+ "role":
245
+ "user",
246
+ "content": [
247
+ {
248
+ "type": "image_url",
249
+ "image_url": {
250
+ "url": image_url
251
+ }
252
+ },
253
+ {
254
+ "type": "image_url",
255
+ "image_url": {
256
+ "url": image_url
257
+ }
258
+ },
259
+ {
260
+ "type": "text",
261
+ "text": "What's in this image?"
262
+ },
263
+ ],
264
+ }]
265
+
266
+ with pytest.raises(openai.BadRequestError): # test multi-image input
267
+ await client.chat.completions.create(
268
+ model=model_name,
269
+ messages=messages,
270
+ max_tokens=10,
271
+ temperature=0.0,
272
+ )
273
+
274
+ # the server should still work afterwards
275
+ completion = await client.completions.create(
276
+ model=model_name,
277
+ prompt=[0, 0, 0, 0, 0],
278
+ max_tokens=5,
279
+ temperature=0.0,
280
+ )
281
+ completion = completion.choices[0].text
282
+ assert completion is not None and len(completion) >= 0
283
+
284
+
285
+if __name__ == "__main__":
286
+ pytest.main([__file__])
tests/multimodal/test_utils.py
new
+75
@@ -0,0 +1,75 @@
1
+import base64
2
+import mimetypes
3
+from tempfile import NamedTemporaryFile
4
+from typing import Dict, Tuple
5
+
6
+import numpy as np
7
+import pytest
8
+import pytest_asyncio
9
+from PIL import Image
10
+
11
+from vllm.multimodal.utils import ImageFetchAiohttp
12
+
13
+# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
14
+TEST_IMAGE_URLS = [
15
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
16
+ "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png",
17
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Venn_diagram_rgb.svg/1280px-Venn_diagram_rgb.svg.png",
18
+ "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png",
19
+]
20
+
21
+
22
+@pytest_asyncio.fixture(scope="session")
23
+async def url_images() -> Dict[str, Image.Image]:
24
+ return {
25
+ image_url: await ImageFetchAiohttp.fetch_image(image_url)
26
+ for image_url in TEST_IMAGE_URLS
27
+ }
28
+
29
+
30
+def get_supported_suffixes() -> Tuple[str, ...]:
31
+ # We should at least test the file types mentioned in GPT-4 with Vision
32
+ OPENAI_SUPPORTED_SUFFIXES = ('.png', '.jpeg', '.jpg', '.webp', '.gif')
33
+
34
+ # Additional file types that are supported by us
35
+ EXTRA_SUPPORTED_SUFFIXES = ('.bmp', '.tiff')
36
+
37
+ return OPENAI_SUPPORTED_SUFFIXES + EXTRA_SUPPORTED_SUFFIXES
38
+
39
+
40
+def _image_equals(a: Image.Image, b: Image.Image) -> bool:
41
+ return (np.asarray(a) == np.asarray(b.convert(a.mode))).all()
42
+
43
+
44
+@pytest.mark.asyncio
45
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
46
+@pytest.mark.parametrize("suffix", get_supported_suffixes())
47
+async def test_fetch_image_base64(url_images: Dict[str, Image.Image],
48
+ image_url: str, suffix: str):
49
+ url_image = url_images[image_url]
50
+
51
+ try:
52
+ mime_type = Image.MIME[Image.registered_extensions()[suffix]]
53
+ except KeyError:
54
+ try:
55
+ mime_type = mimetypes.types_map[suffix]
56
+ except KeyError:
57
+ pytest.skip('No MIME type')
58
+
59
+ with NamedTemporaryFile(suffix=suffix) as f:
60
+ try:
61
+ url_image.save(f.name)
62
+ except Exception as e:
63
+ if e.args[0] == 'cannot write mode RGBA as JPEG':
64
+ pytest.skip('Conversion not supported')
65
+
66
+ raise
67
+
68
+ base64_image = base64.b64encode(f.read()).decode("utf-8")
69
+ data_url = f"data:{mime_type};base64,{base64_image}"
70
+
71
+ data_image = await ImageFetchAiohttp.fetch_image(data_url)
72
+ if _image_equals(url_image, Image.open(f)):
73
+ assert _image_equals(url_image, data_image)
74
+ else:
75
+ pass # Lossy format; only check that image can be opened
vllm/config.py
+11
-1
@@ -5,7 +5,7 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
5
Union)
6
7
import torch
8
-from transformers import PretrainedConfig
8
+from transformers import PretrainedConfig, PreTrainedTokenizerBase
9
10
from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -1119,6 +1119,16 @@ class VisionLanguageConfig:
1119
f"Expecting to choose from "
1120
f"{[x.name for x in cls.ImageInputType]}.") from e
1121
1122
+ #TODO(ywang96): make this a cached property once we refactor the
1123
+ # VisionLanguageConfig class.
1124
+ def get_image_token_text(
1125
+ self, tokenizer: PreTrainedTokenizerBase) -> Tuple[str, str]:
1126
+ """Get the image token placeholder text to be inserted into the
1127
+ text prompt and the string representation of the image token id.
1128
+ """
1129
+ image_token_str = tokenizer.decode(self.image_token_id)
1130
+ return image_token_str * self.image_feature_size, image_token_str
1131
+
1132
def as_cli_args_dict(self) -> Dict[str, Any]:
1133
"""Flatten vision language config to pure args.
1134
vllm/entrypoints/openai/serving_chat.py
+97
-16
@@ -1,15 +1,16 @@
1
import codecs
2
import time
3
-from dataclasses import dataclass
4
-from typing import (AsyncGenerator, AsyncIterator, Dict, Iterable, List,
5
- Optional)
3
+from dataclasses import dataclass, field
4
+from typing import (AsyncGenerator, AsyncIterator, Awaitable, Dict, Iterable,
5
+ List, Optional)
6
from typing import Sequence as GenericSequence
7
from typing import TypedDict, Union, cast, final
8
9
from fastapi import Request
10
-from openai.types.chat import ChatCompletionContentPartTextParam
10
+from openai.types.chat import (ChatCompletionContentPartImageParam,
11
+ ChatCompletionContentPartTextParam)
12
12
-from vllm.config import ModelConfig
13
+from vllm.config import ModelConfig, VisionLanguageConfig
14
from vllm.engine.async_llm_engine import AsyncLLMEngine
15
from vllm.entrypoints.openai.protocol import (
16
ChatCompletionContentPartParam, ChatCompletionLogProb,
@@ -21,9 +22,13 @@ from vllm.entrypoints.openai.protocol import (
22
FunctionCall, ToolCall, UsageInfo)
23
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
24
OpenAIServing)
25
+from vllm.inputs import PromptInputs
26
from vllm.logger import init_logger
27
from vllm.model_executor.guided_decoding import (
28
get_guided_decoding_logits_processor)
29
+from vllm.multimodal.image import ImagePixelData
30
+from vllm.multimodal.utils import (async_get_and_parse_image,
31
+ get_full_image_text_prompt)
32
from vllm.outputs import RequestOutput
33
from vllm.sequence import Logprob
34
from vllm.utils import random_uuid
@@ -40,6 +45,8 @@ class ConversationMessage(TypedDict):
45
@dataclass(frozen=True)
46
class ChatMessageParseResult:
47
messages: List[ConversationMessage]
48
+ image_futures: List[Awaitable[ImagePixelData]] = field(
49
+ default_factory=list)
50
51
52
class OpenAIServingChat(OpenAIServing):
@@ -94,19 +101,76 @@ class OpenAIServingChat(OpenAIServing):
101
parts: Iterable[ChatCompletionContentPartParam],
102
) -> ChatMessageParseResult:
103
texts: List[str] = []
104
+ image_futures: List[Awaitable[ImagePixelData]] = []
105
98
- for _, part in enumerate(parts):
106
+ vlm_config: Optional[VisionLanguageConfig] = getattr(
107
+ self.engine.engine, "vision_language_config", None)
108
+ model_config = getattr(self.engine.engine, "model_config", None)
109
+
110
+ for part in parts:
111
part_type = part["type"]
112
if part_type == "text":
113
text = cast(ChatCompletionContentPartTextParam, part)["text"]
114
115
texts.append(text)
116
+ elif part_type == "image_url":
117
+ if vlm_config is None:
118
+ raise ValueError(
119
+ "'image_url' input is not supported as the loaded "
120
+ "model is not multimodal.")
121
+
122
+ elif len(image_futures) == 0:
123
+ assert self.tokenizer is not None
124
+ image_url = cast(ChatCompletionContentPartImageParam,
125
+ part)["image_url"]
126
+
127
+ if image_url.get("detail", "auto") != "auto":
128
+ logger.warning(
129
+ "'image_url.detail' is currently not supported and "
130
+ "will be ignored.")
131
+
132
+ image_future = async_get_and_parse_image(image_url["url"])
133
+ image_futures.append(image_future)
134
+
135
+ else:
136
+ raise NotImplementedError(
137
+ "Multiple 'image_url' input is currently not supported."
138
+ )
139
+
140
else:
141
raise NotImplementedError(f"Unknown part type: {part_type}")
142
107
- messages = [ConversationMessage(role=role, content="\n".join(texts))]
143
+ text_prompt = "\n".join(texts)
144
+
145
+ if vlm_config is not None and len(image_futures):
146
+
147
+ (image_token_prompt,
148
+ image_token_str) = vlm_config.get_image_token_text(self.tokenizer)
149
109
- return ChatMessageParseResult(messages=messages)
150
+ # NOTE: If image token string (e.g, <image>) is already present
151
+ # in the text prompt, we assume it follows the same format required
152
+ # by the engine.
153
+ if image_token_str in text_prompt:
154
+ logger.warning(
155
+ "Detected image token string in the text prompt. "
156
+ "Skipping prompt formatting.")
157
+ messages = [
158
+ ConversationMessage(role=role, content=text_prompt)
159
+ ]
160
+
161
+ else:
162
+ full_prompt = get_full_image_text_prompt(
163
+ image_prompt=image_token_prompt,
164
+ text_prompt=text_prompt,
165
+ config=model_config)
166
+ messages = [
167
+ ConversationMessage(role=role, content=full_prompt)
168
+ ]
169
+ else:
170
+ messages = [ConversationMessage(role=role, content=text_prompt)]
171
+
172
+ return ChatMessageParseResult(messages=messages,
173
+ image_futures=image_futures)
174
175
def _parse_chat_message_content(
176
self,
@@ -116,10 +180,10 @@ class OpenAIServingChat(OpenAIServing):
180
content = message.get("content")
181
182
if content is None:
119
- return ChatMessageParseResult(messages=[])
183
+ return ChatMessageParseResult(messages=[], image_futures=[])
184
if isinstance(content, str):
185
messages = [ConversationMessage(role=role, content=content)]
122
- return ChatMessageParseResult(messages=messages)
186
+ return ChatMessageParseResult(messages=messages, image_futures=[])
187
188
return self._parse_chat_message_content_parts(role, content)
189
@@ -144,11 +208,13 @@ class OpenAIServingChat(OpenAIServing):
208
209
try:
210
conversation: List[ConversationMessage] = []
211
+ image_futures: List[Awaitable[ImagePixelData]] = []
212
213
for msg in request.messages:
149
- parsed_msg = self._parse_chat_message_content(msg)
214
+ chat_parsed_result = self._parse_chat_message_content(msg)
215
151
- conversation.extend(parsed_msg.messages)
216
+ conversation.extend(chat_parsed_result.messages)
217
+ image_futures.extend(chat_parsed_result.image_futures)
218
219
prompt = self.tokenizer.apply_chat_template(
220
conversation=conversation,
@@ -159,6 +225,17 @@ class OpenAIServingChat(OpenAIServing):
225
logger.error("Error in applying chat template from request: %s", e)
226
return self.create_error_response(str(e))
227
228
+ # Fetch image data
229
+ image_data: Optional[ImagePixelData] = None
230
+ try:
231
+ if len(image_futures):
232
+ # since we support only single image currently
233
+ assert len(image_futures) == 1
234
+ image_data = await image_futures[0]
235
+ except Exception as e:
236
+ logger.error("Error in loading image data: %s", e)
237
+ return self.create_error_response(str(e))
238
+
239
request_id = f"cmpl-{random_uuid()}"
240
try:
241
# Tokenize/detokenize depending on prompt format (string/token list)
@@ -183,11 +260,15 @@ class OpenAIServingChat(OpenAIServing):
260
except ValueError as e:
261
return self.create_error_response(str(e))
262
263
+ inputs: PromptInputs = {
264
+ "prompt": prompt_text,
265
+ "prompt_token_ids": prompt_ids,
266
+ }
267
+ if image_data is not None:
268
+ inputs["multi_modal_data"] = image_data
269
+
270
result_generator = self.engine.generate(
187
- {
188
- "prompt": prompt_text,
189
- "prompt_token_ids": prompt_ids
190
- },
271
+ inputs,
272
sampling_params,
273
request_id,
274
lora_request,
vllm/envs.py
+6
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
29
VLLM_CPU_KVCACHE_SPACE: int = 0
30
VLLM_USE_RAY_COMPILED_DAG: bool = False
31
VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
32
+ VLLM_IMAGE_FETCH_TIMEOUT: int = 5
33
VLLM_TARGET_DEVICE: str = "cuda"
34
MAX_JOBS: Optional[str] = None
35
NVCC_THREADS: Optional[str] = None
@@ -216,6 +217,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
217
# Both spawn and fork work
218
"VLLM_WORKER_MULTIPROC_METHOD":
219
lambda: os.getenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn"),
220
+
221
+ # Timeout for fetching images when serving multimodal models
222
+ # Default is 5 seconds
223
+ "VLLM_IMAGE_FETCH_TIMEOUT":
224
+ lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
225
}
226
227
# end-env-vars-definition
vllm/multimodal/utils.py
new
+85
@@ -0,0 +1,85 @@
1
+import base64
2
+from io import BytesIO
3
+from typing import Optional, Union
4
+
5
+import aiohttp
6
+from PIL import Image
7
+
8
+from vllm.config import ModelConfig
9
+from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
10
+from vllm.multimodal.image import ImagePixelData
11
+
12
+
13
+class ImageFetchAiohttp:
14
+ aiohttp_client: Optional[aiohttp.ClientSession] = None
15
+
16
+ @classmethod
17
+ def get_aiohttp_client(cls) -> aiohttp.ClientSession:
18
+ if cls.aiohttp_client is None:
19
+ timeout = aiohttp.ClientTimeout(total=VLLM_IMAGE_FETCH_TIMEOUT)
20
+ connector = aiohttp.TCPConnector()
21
+ cls.aiohttp_client = aiohttp.ClientSession(timeout=timeout,
22
+ connector=connector)
23
+
24
+ return cls.aiohttp_client
25
+
26
+ @classmethod
27
+ async def fetch_image(cls, image_url: str) -> Image.Image:
28
+ """Load PIL image from a url or base64 encoded openai GPT4V format"""
29
+
30
+ if image_url.startswith('http'):
31
+ # Avoid circular import
32
+ from vllm import __version__ as VLLM_VERSION
33
+
34
+ client = cls.get_aiohttp_client()
35
+ headers = {"User-Agent": f"vLLM/{VLLM_VERSION}"}
36
+
37
+ async with client.get(url=image_url, headers=headers) as response:
38
+ response.raise_for_status()
39
+ image_raw = await response.read()
40
+ image = Image.open(BytesIO(image_raw))
41
+
42
+ # Only split once and assume the second part is the base64 encoded image
43
+ elif image_url.startswith('data:image'):
44
+ image = load_image_from_base64(image_url.split(',', 1)[1])
45
+
46
+ else:
47
+ raise ValueError("Invalid image url: A valid image url must start "
48
+ "with either 'data:image' or 'http'.")
49
+
50
+ return image
51
+
52
+
53
+async def async_get_and_parse_image(image_url: str) -> ImagePixelData:
54
+ with await ImageFetchAiohttp.fetch_image(image_url) as image:
55
+ return ImagePixelData(image)
56
+
57
+
58
+def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
59
+ """encode image to base64 format."""
60
+
61
+ buffered = BytesIO()
62
+ if format == 'JPEG':
63
+ image = image.convert('RGB')
64
+ image.save(buffered, format)
65
+ return base64.b64encode(buffered.getvalue()).decode('utf-8')
66
+
67
+
68
+def load_image_from_base64(image: Union[bytes, str]) -> Image.Image:
69
+ """Load image from base64 format."""
70
+ return Image.open(BytesIO(base64.b64decode(image)))
71
+
72
+
73
+# TODO(ywang96): move this to a model registry for preprocessing vision
74
+# language prompts based on the model type.
75
+def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
76
+ config: ModelConfig) -> str:
77
+ """Combine image and text prompts for vision language model depending on
78
+ the model architecture."""
79
+
80
+ if config.hf_config.model_type == "llava":
81
+ full_prompt = f"{image_prompt}\n{text_prompt}"
82
+ else:
83
+ raise ValueError(
84
+ f"Unsupported model type: {config.hf_config.model_type}")
85
+ return full_prompt