A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend][VLM] Add support for multiple multi-modal items (#8049)
Roger Wang committed
Aug 31, 2024 at 16:35 UTC
5231f0898e559671c6c8cc48efc53a859fce1841
8 files changed
+524
-136
.buildkite/test-pipeline.yaml
+1
@@ -90,6 +90,7 @@ steps:
90
- pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py
91
- pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
92
- pytest -v -s entrypoints/openai
93
+ - pytest -v -s entrypoints/test_chat_utils.py
94
95
- label: Distributed Tests (4 GPUs) # 10min
96
working_dir: "/vllm-workspace/tests"
examples/openai_vision_api_client.py
+39
@@ -1,7 +1,13 @@
1
"""An example showing how to use vLLM to serve VLMs.
2
3
Launch the vLLM server with the following command:
4
+
5
+(single image inference with Llava)
6
vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
7
+
8
+(multi-image inference with Phi-3.5-vision-instruct)
9
+vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
10
+ --trust-remote-code --limit-mm-per-prompt image=2
11
"""
12
import base64
13
@@ -84,3 +90,36 @@ chat_completion_from_base64 = client.chat.completions.create(
90
91
result = chat_completion_from_base64.choices[0].message.content
92
print(f"Chat completion output:{result}")
93
+
94
+# Multi-image input inference
95
+image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
96
+image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
97
+chat_completion_from_url = client.chat.completions.create(
98
+ messages=[{
99
+ "role":
100
+ "user",
101
+ "content": [
102
+ {
103
+ "type": "text",
104
+ "text": "What are the animals in these images?"
105
+ },
106
+ {
107
+ "type": "image_url",
108
+ "image_url": {
109
+ "url": image_url_duck
110
+ },
111
+ },
112
+ {
113
+ "type": "image_url",
114
+ "image_url": {
115
+ "url": image_url_lion
116
+ },
117
+ },
118
+ ],
119
+ }],
120
+ model=model,
121
+ max_tokens=64,
122
+)
123
+
124
+result = chat_completion_from_url.choices[0].message.content
125
+print(f"Chat completion output:{result}")
tests/entrypoints/openai/test_serving_chat.py
+2
@@ -3,6 +3,7 @@ from contextlib import suppress
3
from dataclasses import dataclass
4
from unittest.mock import MagicMock
5
6
+from vllm.config import MultiModalConfig
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
8
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -20,6 +21,7 @@ class MockModelConfig:
21
max_model_len = 100
22
tokenizer_revision = None
23
embedding_mode = False
24
+ multimodal_config = MultiModalConfig()
25
26
27
@dataclass
tests/entrypoints/openai/test_vision.py
+36
-35
@@ -6,11 +6,10 @@ import pytest_asyncio
6
7
from vllm.multimodal.utils import encode_image_base64, fetch_image
8
9
-from ...utils import VLLM_PATH, RemoteOpenAIServer
9
+from ...utils import RemoteOpenAIServer
10
11
-MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
12
-LLAVA_CHAT_TEMPLATE = VLLM_PATH / "examples/template_llava.jinja"
13
-assert LLAVA_CHAT_TEMPLATE.exists()
11
+MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
12
+MAXIMUM_IMAGES = 2
13
14
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15
TEST_IMAGE_URLS = [
@@ -24,13 +23,9 @@ TEST_IMAGE_URLS = [
23
@pytest.fixture(scope="module")
24
def server():
25
args = [
27
- "--dtype",
28
- "bfloat16",
29
- "--max-model-len",
30
- "4096",
31
- "--enforce-eager",
32
- "--chat-template",
33
- str(LLAVA_CHAT_TEMPLATE),
26
+ "--dtype", "bfloat16", "--max-model-len", "4096", "--max-num-seqs",
27
+ "5", "--enforce-eager", "--trust-remote-code", "--limit-mm-per-prompt",
28
+ f"image={MAXIMUM_IMAGES}"
29
]
30
31
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -84,7 +79,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
79
choice = chat_completion.choices[0]
80
assert choice.finish_reason == "length"
81
assert chat_completion.usage == openai.types.CompletionUsage(
87
- completion_tokens=10, prompt_tokens=596, total_tokens=606)
82
+ completion_tokens=10, prompt_tokens=772, total_tokens=782)
83
84
message = choice.message
85
message = chat_completion.choices[0].message
@@ -139,7 +134,7 @@ async def test_single_chat_session_image_base64encoded(
134
choice = chat_completion.choices[0]
135
assert choice.finish_reason == "length"
136
assert chat_completion.usage == openai.types.CompletionUsage(
142
- completion_tokens=10, prompt_tokens=596, total_tokens=606)
137
+ completion_tokens=10, prompt_tokens=772, total_tokens=782)
138
139
message = choice.message
140
message = chat_completion.choices[0].message
@@ -217,26 +212,22 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
212
213
@pytest.mark.asyncio
214
@pytest.mark.parametrize("model_name", [MODEL_NAME])
220
-@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
215
+@pytest.mark.parametrize(
216
+ "image_urls",
217
+ [TEST_IMAGE_URLS[:i] for i in range(2, len(TEST_IMAGE_URLS))])
218
async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
222
- image_url: str):
219
+ image_urls: List[str]):
220
221
messages = [{
222
"role":
223
"user",
224
"content": [
228
- {
229
- "type": "image_url",
230
- "image_url": {
231
- "url": image_url
232
- }
233
- },
234
- {
225
+ *({
226
"type": "image_url",
227
"image_url": {
228
"url": image_url
229
}
239
- },
230
+ } for image_url in image_urls),
231
{
232
"type": "text",
233
"text": "What's in this image?"
@@ -244,20 +235,30 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
235
],
236
}]
237
247
- with pytest.raises(openai.BadRequestError): # test multi-image input
248
- await client.chat.completions.create(
238
+ if len(image_urls) > MAXIMUM_IMAGES:
239
+ with pytest.raises(openai.BadRequestError): # test multi-image input
240
+ await client.chat.completions.create(
241
+ model=model_name,
242
+ messages=messages,
243
+ max_tokens=10,
244
+ temperature=0.0,
245
+ )
246
+
247
+ # the server should still work afterwards
248
+ completion = await client.completions.create(
249
+ model=model_name,
250
+ prompt=[0, 0, 0, 0, 0],
251
+ max_tokens=5,
252
+ temperature=0.0,
253
+ )
254
+ completion = completion.choices[0].text
255
+ assert completion is not None and len(completion) >= 0
256
+ else:
257
+ chat_completion = await client.chat.completions.create(
258
model=model_name,
259
messages=messages,
260
max_tokens=10,
261
temperature=0.0,
262
)
254
-
255
- # the server should still work afterwards
256
- completion = await client.completions.create(
257
- model=model_name,
258
- prompt=[0, 0, 0, 0, 0],
259
- max_tokens=5,
260
- temperature=0.0,
261
- )
262
- completion = completion.choices[0].text
263
- assert completion is not None and len(completion) >= 0
263
+ message = chat_completion.choices[0].message
264
+ assert message.content is not None and len(message.content) >= 0
tests/entrypoints/test_chat_utils.py
new
+305
@@ -0,0 +1,305 @@
1
+import warnings
2
+
3
+import pytest
4
+from PIL import Image
5
+
6
+from vllm.assets.image import ImageAsset
7
+from vllm.config import ModelConfig
8
+from vllm.entrypoints.chat_utils import parse_chat_messages
9
+from vllm.multimodal.utils import encode_image_base64
10
+from vllm.transformers_utils.tokenizer_group import TokenizerGroup
11
+
12
+PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
13
+
14
+
15
+@pytest.fixture(scope="module")
16
+def phi3v_model_config():
17
+ return ModelConfig(PHI3V_MODEL_ID,
18
+ PHI3V_MODEL_ID,
19
+ tokenizer_mode="auto",
20
+ trust_remote_code=True,
21
+ dtype="bfloat16",
22
+ seed=0,
23
+ limit_mm_per_prompt={
24
+ "image": 2,
25
+ })
26
+
27
+
28
+@pytest.fixture(scope="module")
29
+def phi3v_tokenizer():
30
+ return TokenizerGroup(
31
+ tokenizer_id=PHI3V_MODEL_ID,
32
+ enable_lora=False,
33
+ max_num_seqs=5,
34
+ max_input_length=None,
35
+ )
36
+
37
+
38
+@pytest.fixture(scope="module")
39
+def image_url():
40
+ image = ImageAsset('cherry_blossom')
41
+ base64 = encode_image_base64(image.pil_image)
42
+ return f"data:image/jpeg;base64,{base64}"
43
+
44
+
45
+@pytest.mark.asyncio
46
+async def test_parse_chat_messages_with_image_url(phi3v_model_config,
47
+ phi3v_tokenizer, image_url):
48
+ conversation, mm_future = parse_chat_messages([{
49
+ "role":
50
+ "user",
51
+ "content": [{
52
+ "type": "image_url",
53
+ "image_url": {
54
+ "url": image_url
55
+ }
56
+ }, {
57
+ "type": "text",
58
+ "text": "What's in the image?"
59
+ }]
60
+ }], phi3v_model_config, phi3v_tokenizer)
61
+
62
+ assert conversation == [{
63
+ "role": "user",
64
+ "content": "<|image_1|>\nWhat's in the image?"
65
+ }]
66
+ mm_data = await mm_future
67
+ assert set(mm_data.keys()) == {"image"}
68
+ assert isinstance(mm_data["image"], Image.Image)
69
+
70
+
71
+@pytest.mark.asyncio
72
+async def test_parse_chat_messages_multiple_images(phi3v_model_config,
73
+ phi3v_tokenizer, image_url):
74
+ conversation, mm_future = parse_chat_messages([{
75
+ "role":
76
+ "user",
77
+ "content": [{
78
+ "type": "image_url",
79
+ "image_url": {
80
+ "url": image_url
81
+ }
82
+ }, {
83
+ "type": "image_url",
84
+ "image_url": {
85
+ "url": image_url
86
+ }
87
+ }, {
88
+ "type": "text",
89
+ "text": "What's in these images?"
90
+ }]
91
+ }], phi3v_model_config, phi3v_tokenizer)
92
+
93
+ assert conversation == [{
94
+ "role":
95
+ "user",
96
+ "content":
97
+ "<|image_1|>\n<|image_2|>\nWhat's in these images?"
98
+ }]
99
+ mm_data = await mm_future
100
+ assert set(mm_data.keys()) == {"image"}
101
+ assert len(mm_data["image"]) == 2
102
+
103
+
104
+@pytest.mark.asyncio
105
+async def test_parse_chat_messages_placeholder_already_in_prompt(
106
+ phi3v_model_config, phi3v_tokenizer, image_url):
107
+ conversation, mm_future = parse_chat_messages([{
108
+ "role":
109
+ "user",
110
+ "content": [{
111
+ "type": "image_url",
112
+ "image_url": {
113
+ "url": image_url
114
+ }
115
+ }, {
116
+ "type": "image_url",
117
+ "image_url": {
118
+ "url": image_url
119
+ }
120
+ }, {
121
+ "type":
122
+ "text",
123
+ "text":
124
+ "What's in <|image_1|> and how does it compare to <|image_2|>?"
125
+ }]
126
+ }], phi3v_model_config, phi3v_tokenizer)
127
+
128
+ assert conversation == [{
129
+ "role":
130
+ "user",
131
+ "content":
132
+ "What's in <|image_1|> and how does it compare to <|image_2|>?"
133
+ }]
134
+ mm_data = await mm_future
135
+ assert set(mm_data.keys()) == {"image"}
136
+ assert len(mm_data["image"]) == 2
137
+
138
+
139
+@pytest.mark.asyncio
140
+async def test_parse_chat_messages_placeholder_one_already_in_prompt(
141
+ phi3v_model_config, phi3v_tokenizer, image_url):
142
+ conversation, mm_future = parse_chat_messages([{
143
+ "role":
144
+ "user",
145
+ "content": [{
146
+ "type": "image_url",
147
+ "image_url": {
148
+ "url": image_url
149
+ }
150
+ }, {
151
+ "type": "image_url",
152
+ "image_url": {
153
+ "url": image_url
154
+ }
155
+ }, {
156
+ "type":
157
+ "text",
158
+ "text":
159
+ "What's in <|image_1|> and how does it compare to the other one?"
160
+ }]
161
+ }], phi3v_model_config, phi3v_tokenizer)
162
+
163
+ assert conversation == [{
164
+ "role":
165
+ "user",
166
+ "content":
167
+ "<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "
168
+ "other one?"
169
+ }]
170
+ mm_data = await mm_future
171
+ assert set(mm_data.keys()) == {"image"}
172
+ assert len(mm_data["image"]) == 2
173
+
174
+
175
+@pytest.mark.asyncio
176
+async def test_parse_chat_messages_multiple_images_across_messages(
177
+ phi3v_model_config, phi3v_tokenizer, image_url):
178
+ conversation, mm_future = parse_chat_messages([{
179
+ "role":
180
+ "user",
181
+ "content": [{
182
+ "type": "image_url",
183
+ "image_url": {
184
+ "url": image_url
185
+ }
186
+ }, {
187
+ "type": "text",
188
+ "text": "What's in this image?"
189
+ }]
190
+ }, {
191
+ "role": "assistant",
192
+ "content": "Some stuff."
193
+ }, {
194
+ "role":
195
+ "user",
196
+ "content": [{
197
+ "type": "image_url",
198
+ "image_url": {
199
+ "url": image_url
200
+ }
201
+ }, {
202
+ "type": "text",
203
+ "text": "What about this one?"
204
+ }]
205
+ }], phi3v_model_config, phi3v_tokenizer)
206
+
207
+ assert conversation == [
208
+ {
209
+ "role": "user",
210
+ "content": "<|image_1|>\nWhat's in this image?"
211
+ },
212
+ {
213
+ "role": "assistant",
214
+ "content": "Some stuff."
215
+ },
216
+ {
217
+ "role": "user",
218
+ "content": "<|image_2|>\nWhat about this one?"
219
+ },
220
+ ]
221
+ mm_data = await mm_future
222
+ assert set(mm_data.keys()) == {"image"}
223
+ assert len(mm_data["image"]) == 2
224
+
225
+
226
+@pytest.mark.asyncio
227
+async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
228
+ phi3v_model_config, phi3v_tokenizer, image_url):
229
+ with warnings.catch_warnings():
230
+ warnings.filterwarnings(
231
+ "ignore",
232
+ message="coroutine 'async_get_and_parse_image' was never awaited")
233
+ with pytest.raises(
234
+ ValueError,
235
+ match="At most 2 image\\(s\\) may be provided in one request\\."
236
+ ):
237
+ parse_chat_messages([{
238
+ "role":
239
+ "user",
240
+ "content": [{
241
+ "type": "image_url",
242
+ "image_url": {
243
+ "url": image_url
244
+ }
245
+ }, {
246
+ "type": "image_url",
247
+ "image_url": {
248
+ "url": image_url
249
+ }
250
+ }, {
251
+ "type": "image_url",
252
+ "image_url": {
253
+ "url": image_url
254
+ }
255
+ }, {
256
+ "type": "text",
257
+ "text": "What's in these images?"
258
+ }]
259
+ }], phi3v_model_config, phi3v_tokenizer)
260
+
261
+
262
+@pytest.mark.asyncio
263
+async def test_parse_chat_messages_rejects_too_many_images_across_messages(
264
+ phi3v_model_config, phi3v_tokenizer, image_url):
265
+ with warnings.catch_warnings():
266
+ warnings.filterwarnings(
267
+ "ignore",
268
+ message="coroutine 'async_get_and_parse_image' was never awaited")
269
+ with pytest.raises(
270
+ ValueError,
271
+ match="At most 2 image\\(s\\) may be provided in one request\\."
272
+ ):
273
+ parse_chat_messages([{
274
+ "role":
275
+ "user",
276
+ "content": [{
277
+ "type": "image_url",
278
+ "image_url": {
279
+ "url": image_url
280
+ }
281
+ }, {
282
+ "type": "text",
283
+ "text": "What's in this image?"
284
+ }]
285
+ }, {
286
+ "role": "assistant",
287
+ "content": "Some stuff."
288
+ }, {
289
+ "role":
290
+ "user",
291
+ "content": [{
292
+ "type": "image_url",
293
+ "image_url": {
294
+ "url": image_url
295
+ }
296
+ }, {
297
+ "type": "image_url",
298
+ "image_url": {
299
+ "url": image_url
300
+ }
301
+ }, {
302
+ "type": "text",
303
+ "text": "What about these two?"
304
+ }]
305
+ }], phi3v_model_config, phi3v_tokenizer)
vllm/entrypoints/chat_utils.py
+136
-92
@@ -1,9 +1,10 @@
1
+import asyncio
2
import codecs
2
-from dataclasses import dataclass
3
+from collections import defaultdict
4
from functools import lru_cache
5
from pathlib import Path
5
-from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6
- Union)
6
+from typing import (Any, Awaitable, Dict, Iterable, List, Literal, Mapping,
7
+ Optional, Tuple, Union)
8
9
# yapf conflicts with isort for this block
10
# yapf: disable
@@ -80,10 +81,90 @@ class ConversationMessage(TypedDict):
81
content: str
82
83
83
-@dataclass(frozen=True)
84
-class ChatMessageParseResult:
85
- messages: List[ConversationMessage]
86
- mm_futures: List[Awaitable[MultiModalDataDict]]
84
+class MultiModalItemTracker:
85
+ """
86
+ Tracks multi-modal items in a given request and ensures that the number
87
+ of multi-modal items in a given request does not exceed the configured
88
+ maximum per prompt.
89
+ """
90
+
91
+ def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
92
+ self._model_config = model_config
93
+ self._tokenizer = tokenizer
94
+ self._allowed_items = (model_config.multimodal_config.limit_per_prompt
95
+ if model_config.multimodal_config else {})
96
+ self._consumed_items = {k: 0 for k in self._allowed_items}
97
+ self._futures: List[Awaitable[MultiModalDataDict]] = []
98
+
99
+ @staticmethod
100
+ @lru_cache(maxsize=None)
101
+ def _cached_token_str(tokenizer: AnyTokenizer, token_index: int):
102
+ return tokenizer.decode(token_index)
103
+
104
+ def add(self, modality: Literal["image", "audio"],
105
+ mm_future: Awaitable[MultiModalDataDict]) -> Optional[str]:
106
+ """
107
+ Adds the multi-modal item to the current prompt and returns the
108
+ placeholder string to use, if any.
109
+ """
110
+ allowed_count = self._allowed_items.get(modality, 1)
111
+ current_count = self._consumed_items.get(modality, 0) + 1
112
+ if current_count > allowed_count:
113
+ raise ValueError(
114
+ f"At most {allowed_count} {modality}(s) may be provided in "
115
+ "one request.")
116
+
117
+ self._consumed_items[modality] = current_count
118
+ self._futures.append(mm_future)
119
+
120
+ # TODO: Let user specify how to insert image tokens into prompt
121
+ # (similar to chat template)
122
+ model_type = self._model_config.hf_config.model_type
123
+ if modality == "image":
124
+ if model_type == "phi3_v":
125
+ # Workaround since this token is not defined in the tokenizer
126
+ return f"<|image_{current_count}|>"
127
+ if model_type == "minicpmv":
128
+ return "(<image>./</image>)"
129
+ if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
130
+ # These models do not use image tokens in the prompt
131
+ return None
132
+ if model_type.startswith("llava"):
133
+ return MultiModalItemTracker._cached_token_str(
134
+ self._tokenizer,
135
+ self._model_config.hf_config.image_token_index)
136
+ if model_type in ("chameleon", "internvl_chat"):
137
+ return "<image>"
138
+
139
+ raise TypeError(f"Unknown model type: {model_type}")
140
+ elif modality == "audio":
141
+ if model_type == "ultravox":
142
+ return "<|reserved_special_token_0|>"
143
+ raise TypeError(f"Unknown model type: {model_type}")
144
+ else:
145
+ raise TypeError(f"Unknown modality: {modality}")
146
+
147
+ @staticmethod
148
+ async def _combine(futures: List[Awaitable[MultiModalDataDict]]):
149
+ mm_lists: Mapping[str, List[object]] = defaultdict(list)
150
+
151
+ # Merge all the multi-modal items
152
+ for single_mm_data in (await asyncio.gather(*futures)):
153
+ for mm_key, mm_item in single_mm_data.items():
154
+ if isinstance(mm_item, list):
155
+ mm_lists[mm_key].extend(mm_item)
156
+ else:
157
+ mm_lists[mm_key].append(mm_item)
158
+
159
+ # Unpack any single item lists for models that don't expect multiple.
160
+ return {
161
+ mm_key: mm_list[0] if len(mm_list) == 1 else mm_list
162
+ for mm_key, mm_list in mm_lists.items()
163
+ }
164
+
165
+ def all_mm_data(self) -> Optional[Awaitable[MultiModalDataDict]]:
166
+ return MultiModalItemTracker._combine(
167
+ self._futures) if self._futures else None
168
169
170
def load_chat_template(
@@ -112,44 +193,30 @@ def load_chat_template(
193
return resolved_chat_template
194
195
115
-@lru_cache(maxsize=None)
116
-def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117
- modality: Literal["image", "audio"]) -> Optional[str]:
118
- # TODO: Let user specify how to insert image tokens into prompt
119
- # (similar to chat template)
120
- model_type = model_config.hf_config.model_type
121
- if modality == "image":
122
- if model_type == "phi3_v":
123
- # Workaround since this token is not defined in the tokenizer
124
- return "<|image_1|>"
125
- if model_type == "minicpmv":
126
- return "(<image>./</image>)"
127
- if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
128
- # These models do not use image tokens in the prompt
129
- return None
130
- if model_type.startswith("llava"):
131
- return tokenizer.decode(model_config.hf_config.image_token_index)
132
- if model_type in ("chameleon", "internvl_chat"):
133
- return "<image>"
134
-
135
- raise TypeError(f"Unknown model type: {model_type}")
136
- elif modality == "audio":
137
- if model_type == "ultravox":
138
- return "<|reserved_special_token_0|>"
139
- raise TypeError(f"Unknown model type: {model_type}")
140
- else:
141
- raise TypeError(f"Unknown modality: {modality}")
142
-
143
-
196
# TODO: Let user specify how to insert multimodal tokens into prompt
197
# (similar to chat template)
146
-def _get_full_multimodal_text_prompt(placeholder_token_str: str,
198
+def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
199
text_prompt: str) -> str:
200
"""Combine multimodal prompts for a multimodal language model"""
201
150
- # NOTE: For now we assume all model architectures use the same
151
- # placeholder + text prompt format. This may change in the future.
152
- return f"{placeholder_token_str}\n{text_prompt}"
202
+ # Look through the text prompt to check for missing placeholders
203
+ missing_placeholders = []
204
+ for placeholder in placeholder_counts:
205
+
206
+ # For any existing placeholder in the text prompt, we leave it as is
207
+ placeholder_counts[placeholder] -= text_prompt.count(placeholder)
208
+
209
+ if placeholder_counts[placeholder] < 0:
210
+ raise ValueError(
211
+ f"Found more '{placeholder}' placeholders in input prompt than "
212
+ "actual multimodal data items.")
213
+
214
+ missing_placeholders.extend([placeholder] *
215
+ placeholder_counts[placeholder])
216
+
217
+ # NOTE: For now we always add missing placeholders at the front of
218
+ # the prompt. This may change to be customizable in the future.
219
+ return "\n".join(missing_placeholders + [text_prompt])
220
221
222
_TextParser = TypeAdapter(ChatCompletionContentPartTextParam)
@@ -160,12 +227,12 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
227
def _parse_chat_message_content_parts(
228
role: str,
229
parts: Iterable[ChatCompletionContentPartParam],
163
- model_config: ModelConfig,
164
- tokenizer: AnyTokenizer,
165
-) -> ChatMessageParseResult:
230
+ mm_tracker: MultiModalItemTracker,
231
+) -> List[ConversationMessage]:
232
texts: List[str] = []
167
- mm_futures: List[Awaitable[MultiModalDataDict]] = []
168
- modality: Literal["image", "audio"] = "image"
233
+
234
+ # multimodal placeholder_string : count
235
+ mm_placeholder_counts: Dict[str, int] = {}
236
237
for part in parts:
238
part_type = part["type"]
@@ -173,11 +240,6 @@ def _parse_chat_message_content_parts(
240
text = _TextParser.validate_python(part)["text"]
241
texts.append(text)
242
elif part_type == "image_url":
176
- modality = "image"
177
- if len(mm_futures) > 0:
178
- raise NotImplementedError(
179
- "Multiple multimodal inputs is currently not supported.")
180
-
243
image_url = _ImageParser.validate_python(part)["image_url"]
244
245
if image_url.get("detail", "auto") != "auto":
@@ -185,60 +247,44 @@ def _parse_chat_message_content_parts(
247
"'image_url.detail' is currently not supported and "
248
"will be ignored.")
249
188
- image_future = async_get_and_parse_image(image_url["url"])
189
- mm_futures.append(image_future)
250
+ image_coro = async_get_and_parse_image(image_url["url"])
251
+ placeholder = mm_tracker.add("image", image_coro)
252
+ if placeholder:
253
+ mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
254
+ placeholder, 0) + 1
255
elif part_type == "audio_url":
191
- modality = "audio"
192
- if len(mm_futures) > 0:
193
- raise NotImplementedError(
194
- "Multiple multimodal inputs is currently not supported.")
195
-
256
audio_url = _AudioParser.validate_python(part)["audio_url"]
197
- audio_future = async_get_and_parse_audio(audio_url["url"])
198
- mm_futures.append(audio_future)
257
+ audio_coro = async_get_and_parse_audio(audio_url["url"])
258
+ placeholder = mm_tracker.add("audio", audio_coro)
259
+ if placeholder:
260
+ mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
261
+ placeholder, 0) + 1
262
else:
263
raise NotImplementedError(f"Unknown part type: {part_type}")
264
265
text_prompt = "\n".join(texts)
266
+ if mm_placeholder_counts:
267
+ text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
268
+ text_prompt)
269
204
- if mm_futures:
205
- placeholder_token_str = _mm_token_str(model_config, tokenizer,
206
- modality)
207
- if placeholder_token_str is not None:
208
- if placeholder_token_str in text_prompt:
209
- logger.warning(
210
- "Detected multi-modal token string in the text prompt. "
211
- "Skipping prompt formatting.")
212
- else:
213
- text_prompt = _get_full_multimodal_text_prompt(
214
- placeholder_token_str=placeholder_token_str,
215
- text_prompt=text_prompt,
216
- )
217
-
218
- messages = [ConversationMessage(role=role, content=text_prompt)]
219
-
220
- return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
270
+ return [ConversationMessage(role=role, content=text_prompt)]
271
272
273
def _parse_chat_message_content(
224
- message: ChatCompletionMessageParam,
225
- model_config: ModelConfig,
226
- tokenizer: AnyTokenizer,
227
-) -> ChatMessageParseResult:
274
+ message: ChatCompletionMessageParam,
275
+ mm_tracker: MultiModalItemTracker) -> List[ConversationMessage]:
276
role = message["role"]
277
content = message.get("content")
278
279
if content is None:
232
- return ChatMessageParseResult(messages=[], mm_futures=[])
280
+ return []
281
if isinstance(content, str):
234
- messages = [ConversationMessage(role=role, content=content)]
235
- return ChatMessageParseResult(messages=messages, mm_futures=[])
282
+ return [ConversationMessage(role=role, content=content)]
283
284
return _parse_chat_message_content_parts(
285
role,
286
content, # type: ignore
240
- model_config,
241
- tokenizer,
287
+ mm_tracker,
288
)
289
290
@@ -246,18 +292,16 @@ def parse_chat_messages(
292
messages: List[ChatCompletionMessageParam],
293
model_config: ModelConfig,
294
tokenizer: AnyTokenizer,
249
-) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
295
+) -> Tuple[List[ConversationMessage], Optional[Awaitable[MultiModalDataDict]]]:
296
conversation: List[ConversationMessage] = []
251
- mm_futures: List[Awaitable[MultiModalDataDict]] = []
297
+ mm_tracker = MultiModalItemTracker(model_config, tokenizer)
298
299
for msg in messages:
254
- parse_result = _parse_chat_message_content(msg, model_config,
255
- tokenizer)
300
+ sub_messages = _parse_chat_message_content(msg, mm_tracker)
301
257
- conversation.extend(parse_result.messages)
258
- mm_futures.extend(parse_result.mm_futures)
302
+ conversation.extend(sub_messages)
303
260
- return conversation, mm_futures
304
+ return conversation, mm_tracker.all_mm_data()
305
306
307
def apply_chat_template(
vllm/entrypoints/openai/serving_chat.py
+3
-7
@@ -94,7 +94,7 @@ class OpenAIServingChat(OpenAIServing):
94
tokenizer = await self.async_engine_client.get_tokenizer(
95
lora_request)
96
97
- conversation, mm_futures = parse_chat_messages(
97
+ conversation, mm_data_future = parse_chat_messages(
98
request.messages, model_config, tokenizer)
99
100
tool_dicts = None if request.tools is None else [
@@ -116,12 +116,8 @@ class OpenAIServingChat(OpenAIServing):
116
117
mm_data: Optional[MultiModalDataDict] = None
118
try:
119
- if len(mm_futures):
120
- # since we support only single mm data currently
121
- assert len(
122
- mm_futures
123
- ) == 1, "Multiple 'image_url' input is currently not supported."
124
- mm_data = await mm_futures[0]
119
+ if mm_data_future:
120
+ mm_data = await mm_data_future
121
except Exception as e:
122
logger.error("Error in loading multi-modal data: %s", e)
123
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+2
-2
@@ -65,10 +65,10 @@ class OpenAIServingTokenization(OpenAIServing):
65
if isinstance(request, TokenizeChatRequest):
66
model_config = self.model_config
67
68
- conversation, mm_futures = parse_chat_messages(
68
+ conversation, mm_data_future = parse_chat_messages(
69
request.messages, model_config, tokenizer)
70
71
- if mm_futures:
71
+ if mm_data_future:
72
logger.warning(
73
"Multi-modal inputs are ignored during tokenization")
74