@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5231f0898e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend][VLM] Add support for multiple multi-modal items (#8049)

Roger Wang committed Aug 31, 2024 at 16:35 UTC 5231f0898e559671c6c8cc48efc53a859fce1841
8 files changed +524 -136
.buildkite/test-pipeline.yaml
+1
@@ -90,6 +90,7 @@ steps:
90 - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py
91 - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
92 - pytest -v -s entrypoints/openai
93 + - pytest -v -s entrypoints/test_chat_utils.py
94
95 - label: Distributed Tests (4 GPUs) # 10min
96 working_dir: "/vllm-workspace/tests"
examples/openai_vision_api_client.py
+39
@@ -1,7 +1,13 @@
1 """An example showing how to use vLLM to serve VLMs.
2
3 Launch the vLLM server with the following command:
4 +
5 +(single image inference with Llava)
6 vllm serve llava-hf/llava-1.5-7b-hf --chat-template template_llava.jinja
7 +
8 +(multi-image inference with Phi-3.5-vision-instruct)
9 +vllm serve microsoft/Phi-3.5-vision-instruct --max-model-len 4096 \
10 + --trust-remote-code --limit-mm-per-prompt image=2
11 """
12 import base64
13
@@ -84,3 +90,36 @@ chat_completion_from_base64 = client.chat.completions.create(
90
91 result = chat_completion_from_base64.choices[0].message.content
92 print(f"Chat completion output:{result}")
93 +
94 +# Multi-image input inference
95 +image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
96 +image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
97 +chat_completion_from_url = client.chat.completions.create(
98 + messages=[{
99 + "role":
100 + "user",
101 + "content": [
102 + {
103 + "type": "text",
104 + "text": "What are the animals in these images?"
105 + },
106 + {
107 + "type": "image_url",
108 + "image_url": {
109 + "url": image_url_duck
110 + },
111 + },
112 + {
113 + "type": "image_url",
114 + "image_url": {
115 + "url": image_url_lion
116 + },
117 + },
118 + ],
119 + }],
120 + model=model,
121 + max_tokens=64,
122 +)
123 +
124 +result = chat_completion_from_url.choices[0].message.content
125 +print(f"Chat completion output:{result}")
tests/entrypoints/openai/test_serving_chat.py
+2
@@ -3,6 +3,7 @@ from contextlib import suppress
3 from dataclasses import dataclass
4 from unittest.mock import MagicMock
5
6 +from vllm.config import MultiModalConfig
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
8 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
9 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
@@ -20,6 +21,7 @@ class MockModelConfig:
21 max_model_len = 100
22 tokenizer_revision = None
23 embedding_mode = False
24 + multimodal_config = MultiModalConfig()
25
26
27 @dataclass
tests/entrypoints/openai/test_vision.py
+36 -35
@@ -6,11 +6,10 @@ import pytest_asyncio
6
7 from vllm.multimodal.utils import encode_image_base64, fetch_image
8
9 -from ...utils import VLLM_PATH, RemoteOpenAIServer
9 +from ...utils import RemoteOpenAIServer
10
11 -MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
12 -LLAVA_CHAT_TEMPLATE = VLLM_PATH / "examples/template_llava.jinja"
13 -assert LLAVA_CHAT_TEMPLATE.exists()
11 +MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
12 +MAXIMUM_IMAGES = 2
13
14 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15 TEST_IMAGE_URLS = [
@@ -24,13 +23,9 @@ TEST_IMAGE_URLS = [
23 @pytest.fixture(scope="module")
24 def server():
25 args = [
27 - "--dtype",
28 - "bfloat16",
29 - "--max-model-len",
30 - "4096",
31 - "--enforce-eager",
32 - "--chat-template",
33 - str(LLAVA_CHAT_TEMPLATE),
26 + "--dtype", "bfloat16", "--max-model-len", "4096", "--max-num-seqs",
27 + "5", "--enforce-eager", "--trust-remote-code", "--limit-mm-per-prompt",
28 + f"image={MAXIMUM_IMAGES}"
29 ]
30
31 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -84,7 +79,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
79 choice = chat_completion.choices[0]
80 assert choice.finish_reason == "length"
81 assert chat_completion.usage == openai.types.CompletionUsage(
87 - completion_tokens=10, prompt_tokens=596, total_tokens=606)
82 + completion_tokens=10, prompt_tokens=772, total_tokens=782)
83
84 message = choice.message
85 message = chat_completion.choices[0].message
@@ -139,7 +134,7 @@ async def test_single_chat_session_image_base64encoded(
134 choice = chat_completion.choices[0]
135 assert choice.finish_reason == "length"
136 assert chat_completion.usage == openai.types.CompletionUsage(
142 - completion_tokens=10, prompt_tokens=596, total_tokens=606)
137 + completion_tokens=10, prompt_tokens=772, total_tokens=782)
138
139 message = choice.message
140 message = chat_completion.choices[0].message
@@ -217,26 +212,22 @@ async def test_chat_streaming_image(client: openai.AsyncOpenAI,
212
213 @pytest.mark.asyncio
214 @pytest.mark.parametrize("model_name", [MODEL_NAME])
220 -@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
215 +@pytest.mark.parametrize(
216 + "image_urls",
217 + [TEST_IMAGE_URLS[:i] for i in range(2, len(TEST_IMAGE_URLS))])
218 async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
222 - image_url: str):
219 + image_urls: List[str]):
220
221 messages = [{
222 "role":
223 "user",
224 "content": [
228 - {
229 - "type": "image_url",
230 - "image_url": {
231 - "url": image_url
232 - }
233 - },
234 - {
225 + *({
226 "type": "image_url",
227 "image_url": {
228 "url": image_url
229 }
239 - },
230 + } for image_url in image_urls),
231 {
232 "type": "text",
233 "text": "What's in this image?"
@@ -244,20 +235,30 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
235 ],
236 }]
237
247 - with pytest.raises(openai.BadRequestError): # test multi-image input
248 - await client.chat.completions.create(
238 + if len(image_urls) > MAXIMUM_IMAGES:
239 + with pytest.raises(openai.BadRequestError): # test multi-image input
240 + await client.chat.completions.create(
241 + model=model_name,
242 + messages=messages,
243 + max_tokens=10,
244 + temperature=0.0,
245 + )
246 +
247 + # the server should still work afterwards
248 + completion = await client.completions.create(
249 + model=model_name,
250 + prompt=[0, 0, 0, 0, 0],
251 + max_tokens=5,
252 + temperature=0.0,
253 + )
254 + completion = completion.choices[0].text
255 + assert completion is not None and len(completion) >= 0
256 + else:
257 + chat_completion = await client.chat.completions.create(
258 model=model_name,
259 messages=messages,
260 max_tokens=10,
261 temperature=0.0,
262 )
254 -
255 - # the server should still work afterwards
256 - completion = await client.completions.create(
257 - model=model_name,
258 - prompt=[0, 0, 0, 0, 0],
259 - max_tokens=5,
260 - temperature=0.0,
261 - )
262 - completion = completion.choices[0].text
263 - assert completion is not None and len(completion) >= 0
263 + message = chat_completion.choices[0].message
264 + assert message.content is not None and len(message.content) >= 0
tests/entrypoints/test_chat_utils.py new
+305
@@ -0,0 +1,305 @@
1 +import warnings
2 +
3 +import pytest
4 +from PIL import Image
5 +
6 +from vllm.assets.image import ImageAsset
7 +from vllm.config import ModelConfig
8 +from vllm.entrypoints.chat_utils import parse_chat_messages
9 +from vllm.multimodal.utils import encode_image_base64
10 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
11 +
12 +PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
13 +
14 +
15 +@pytest.fixture(scope="module")
16 +def phi3v_model_config():
17 + return ModelConfig(PHI3V_MODEL_ID,
18 + PHI3V_MODEL_ID,
19 + tokenizer_mode="auto",
20 + trust_remote_code=True,
21 + dtype="bfloat16",
22 + seed=0,
23 + limit_mm_per_prompt={
24 + "image": 2,
25 + })
26 +
27 +
28 +@pytest.fixture(scope="module")
29 +def phi3v_tokenizer():
30 + return TokenizerGroup(
31 + tokenizer_id=PHI3V_MODEL_ID,
32 + enable_lora=False,
33 + max_num_seqs=5,
34 + max_input_length=None,
35 + )
36 +
37 +
38 +@pytest.fixture(scope="module")
39 +def image_url():
40 + image = ImageAsset('cherry_blossom')
41 + base64 = encode_image_base64(image.pil_image)
42 + return f"data:image/jpeg;base64,{base64}"
43 +
44 +
45 +@pytest.mark.asyncio
46 +async def test_parse_chat_messages_with_image_url(phi3v_model_config,
47 + phi3v_tokenizer, image_url):
48 + conversation, mm_future = parse_chat_messages([{
49 + "role":
50 + "user",
51 + "content": [{
52 + "type": "image_url",
53 + "image_url": {
54 + "url": image_url
55 + }
56 + }, {
57 + "type": "text",
58 + "text": "What's in the image?"
59 + }]
60 + }], phi3v_model_config, phi3v_tokenizer)
61 +
62 + assert conversation == [{
63 + "role": "user",
64 + "content": "<|image_1|>\nWhat's in the image?"
65 + }]
66 + mm_data = await mm_future
67 + assert set(mm_data.keys()) == {"image"}
68 + assert isinstance(mm_data["image"], Image.Image)
69 +
70 +
71 +@pytest.mark.asyncio
72 +async def test_parse_chat_messages_multiple_images(phi3v_model_config,
73 + phi3v_tokenizer, image_url):
74 + conversation, mm_future = parse_chat_messages([{
75 + "role":
76 + "user",
77 + "content": [{
78 + "type": "image_url",
79 + "image_url": {
80 + "url": image_url
81 + }
82 + }, {
83 + "type": "image_url",
84 + "image_url": {
85 + "url": image_url
86 + }
87 + }, {
88 + "type": "text",
89 + "text": "What's in these images?"
90 + }]
91 + }], phi3v_model_config, phi3v_tokenizer)
92 +
93 + assert conversation == [{
94 + "role":
95 + "user",
96 + "content":
97 + "<|image_1|>\n<|image_2|>\nWhat's in these images?"
98 + }]
99 + mm_data = await mm_future
100 + assert set(mm_data.keys()) == {"image"}
101 + assert len(mm_data["image"]) == 2
102 +
103 +
104 +@pytest.mark.asyncio
105 +async def test_parse_chat_messages_placeholder_already_in_prompt(
106 + phi3v_model_config, phi3v_tokenizer, image_url):
107 + conversation, mm_future = parse_chat_messages([{
108 + "role":
109 + "user",
110 + "content": [{
111 + "type": "image_url",
112 + "image_url": {
113 + "url": image_url
114 + }
115 + }, {
116 + "type": "image_url",
117 + "image_url": {
118 + "url": image_url
119 + }
120 + }, {
121 + "type":
122 + "text",
123 + "text":
124 + "What's in <|image_1|> and how does it compare to <|image_2|>?"
125 + }]
126 + }], phi3v_model_config, phi3v_tokenizer)
127 +
128 + assert conversation == [{
129 + "role":
130 + "user",
131 + "content":
132 + "What's in <|image_1|> and how does it compare to <|image_2|>?"
133 + }]
134 + mm_data = await mm_future
135 + assert set(mm_data.keys()) == {"image"}
136 + assert len(mm_data["image"]) == 2
137 +
138 +
139 +@pytest.mark.asyncio
140 +async def test_parse_chat_messages_placeholder_one_already_in_prompt(
141 + phi3v_model_config, phi3v_tokenizer, image_url):
142 + conversation, mm_future = parse_chat_messages([{
143 + "role":
144 + "user",
145 + "content": [{
146 + "type": "image_url",
147 + "image_url": {
148 + "url": image_url
149 + }
150 + }, {
151 + "type": "image_url",
152 + "image_url": {
153 + "url": image_url
154 + }
155 + }, {
156 + "type":
157 + "text",
158 + "text":
159 + "What's in <|image_1|> and how does it compare to the other one?"
160 + }]
161 + }], phi3v_model_config, phi3v_tokenizer)
162 +
163 + assert conversation == [{
164 + "role":
165 + "user",
166 + "content":
167 + "<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "
168 + "other one?"
169 + }]
170 + mm_data = await mm_future
171 + assert set(mm_data.keys()) == {"image"}
172 + assert len(mm_data["image"]) == 2
173 +
174 +
175 +@pytest.mark.asyncio
176 +async def test_parse_chat_messages_multiple_images_across_messages(
177 + phi3v_model_config, phi3v_tokenizer, image_url):
178 + conversation, mm_future = parse_chat_messages([{
179 + "role":
180 + "user",
181 + "content": [{
182 + "type": "image_url",
183 + "image_url": {
184 + "url": image_url
185 + }
186 + }, {
187 + "type": "text",
188 + "text": "What's in this image?"
189 + }]
190 + }, {
191 + "role": "assistant",
192 + "content": "Some stuff."
193 + }, {
194 + "role":
195 + "user",
196 + "content": [{
197 + "type": "image_url",
198 + "image_url": {
199 + "url": image_url
200 + }
201 + }, {
202 + "type": "text",
203 + "text": "What about this one?"
204 + }]
205 + }], phi3v_model_config, phi3v_tokenizer)
206 +
207 + assert conversation == [
208 + {
209 + "role": "user",
210 + "content": "<|image_1|>\nWhat's in this image?"
211 + },
212 + {
213 + "role": "assistant",
214 + "content": "Some stuff."
215 + },
216 + {
217 + "role": "user",
218 + "content": "<|image_2|>\nWhat about this one?"
219 + },
220 + ]
221 + mm_data = await mm_future
222 + assert set(mm_data.keys()) == {"image"}
223 + assert len(mm_data["image"]) == 2
224 +
225 +
226 +@pytest.mark.asyncio
227 +async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
228 + phi3v_model_config, phi3v_tokenizer, image_url):
229 + with warnings.catch_warnings():
230 + warnings.filterwarnings(
231 + "ignore",
232 + message="coroutine 'async_get_and_parse_image' was never awaited")
233 + with pytest.raises(
234 + ValueError,
235 + match="At most 2 image\\(s\\) may be provided in one request\\."
236 + ):
237 + parse_chat_messages([{
238 + "role":
239 + "user",
240 + "content": [{
241 + "type": "image_url",
242 + "image_url": {
243 + "url": image_url
244 + }
245 + }, {
246 + "type": "image_url",
247 + "image_url": {
248 + "url": image_url
249 + }
250 + }, {
251 + "type": "image_url",
252 + "image_url": {
253 + "url": image_url
254 + }
255 + }, {
256 + "type": "text",
257 + "text": "What's in these images?"
258 + }]
259 + }], phi3v_model_config, phi3v_tokenizer)
260 +
261 +
262 +@pytest.mark.asyncio
263 +async def test_parse_chat_messages_rejects_too_many_images_across_messages(
264 + phi3v_model_config, phi3v_tokenizer, image_url):
265 + with warnings.catch_warnings():
266 + warnings.filterwarnings(
267 + "ignore",
268 + message="coroutine 'async_get_and_parse_image' was never awaited")
269 + with pytest.raises(
270 + ValueError,
271 + match="At most 2 image\\(s\\) may be provided in one request\\."
272 + ):
273 + parse_chat_messages([{
274 + "role":
275 + "user",
276 + "content": [{
277 + "type": "image_url",
278 + "image_url": {
279 + "url": image_url
280 + }
281 + }, {
282 + "type": "text",
283 + "text": "What's in this image?"
284 + }]
285 + }, {
286 + "role": "assistant",
287 + "content": "Some stuff."
288 + }, {
289 + "role":
290 + "user",
291 + "content": [{
292 + "type": "image_url",
293 + "image_url": {
294 + "url": image_url
295 + }
296 + }, {
297 + "type": "image_url",
298 + "image_url": {
299 + "url": image_url
300 + }
301 + }, {
302 + "type": "text",
303 + "text": "What about these two?"
304 + }]
305 + }], phi3v_model_config, phi3v_tokenizer)
vllm/entrypoints/chat_utils.py
+136 -92
@@ -1,9 +1,10 @@
1 +import asyncio
2 import codecs
2 -from dataclasses import dataclass
3 +from collections import defaultdict
4 from functools import lru_cache
5 from pathlib import Path
5 -from typing import (Any, Awaitable, Iterable, List, Literal, Optional, Tuple,
6 - Union)
6 +from typing import (Any, Awaitable, Dict, Iterable, List, Literal, Mapping,
7 + Optional, Tuple, Union)
8
9 # yapf conflicts with isort for this block
10 # yapf: disable
@@ -80,10 +81,90 @@ class ConversationMessage(TypedDict):
81 content: str
82
83
83 -@dataclass(frozen=True)
84 -class ChatMessageParseResult:
85 - messages: List[ConversationMessage]
86 - mm_futures: List[Awaitable[MultiModalDataDict]]
84 +class MultiModalItemTracker:
85 + """
86 + Tracks multi-modal items in a given request and ensures that the number
87 + of multi-modal items in a given request does not exceed the configured
88 + maximum per prompt.
89 + """
90 +
91 + def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
92 + self._model_config = model_config
93 + self._tokenizer = tokenizer
94 + self._allowed_items = (model_config.multimodal_config.limit_per_prompt
95 + if model_config.multimodal_config else {})
96 + self._consumed_items = {k: 0 for k in self._allowed_items}
97 + self._futures: List[Awaitable[MultiModalDataDict]] = []
98 +
99 + @staticmethod
100 + @lru_cache(maxsize=None)
101 + def _cached_token_str(tokenizer: AnyTokenizer, token_index: int):
102 + return tokenizer.decode(token_index)
103 +
104 + def add(self, modality: Literal["image", "audio"],
105 + mm_future: Awaitable[MultiModalDataDict]) -> Optional[str]:
106 + """
107 + Adds the multi-modal item to the current prompt and returns the
108 + placeholder string to use, if any.
109 + """
110 + allowed_count = self._allowed_items.get(modality, 1)
111 + current_count = self._consumed_items.get(modality, 0) + 1
112 + if current_count > allowed_count:
113 + raise ValueError(
114 + f"At most {allowed_count} {modality}(s) may be provided in "
115 + "one request.")
116 +
117 + self._consumed_items[modality] = current_count
118 + self._futures.append(mm_future)
119 +
120 + # TODO: Let user specify how to insert image tokens into prompt
121 + # (similar to chat template)
122 + model_type = self._model_config.hf_config.model_type
123 + if modality == "image":
124 + if model_type == "phi3_v":
125 + # Workaround since this token is not defined in the tokenizer
126 + return f"<|image_{current_count}|>"
127 + if model_type == "minicpmv":
128 + return "(<image>./</image>)"
129 + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
130 + # These models do not use image tokens in the prompt
131 + return None
132 + if model_type.startswith("llava"):
133 + return MultiModalItemTracker._cached_token_str(
134 + self._tokenizer,
135 + self._model_config.hf_config.image_token_index)
136 + if model_type in ("chameleon", "internvl_chat"):
137 + return "<image>"
138 +
139 + raise TypeError(f"Unknown model type: {model_type}")
140 + elif modality == "audio":
141 + if model_type == "ultravox":
142 + return "<|reserved_special_token_0|>"
143 + raise TypeError(f"Unknown model type: {model_type}")
144 + else:
145 + raise TypeError(f"Unknown modality: {modality}")
146 +
147 + @staticmethod
148 + async def _combine(futures: List[Awaitable[MultiModalDataDict]]):
149 + mm_lists: Mapping[str, List[object]] = defaultdict(list)
150 +
151 + # Merge all the multi-modal items
152 + for single_mm_data in (await asyncio.gather(*futures)):
153 + for mm_key, mm_item in single_mm_data.items():
154 + if isinstance(mm_item, list):
155 + mm_lists[mm_key].extend(mm_item)
156 + else:
157 + mm_lists[mm_key].append(mm_item)
158 +
159 + # Unpack any single item lists for models that don't expect multiple.
160 + return {
161 + mm_key: mm_list[0] if len(mm_list) == 1 else mm_list
162 + for mm_key, mm_list in mm_lists.items()
163 + }
164 +
165 + def all_mm_data(self) -> Optional[Awaitable[MultiModalDataDict]]:
166 + return MultiModalItemTracker._combine(
167 + self._futures) if self._futures else None
168
169
170 def load_chat_template(
@@ -112,44 +193,30 @@ def load_chat_template(
193 return resolved_chat_template
194
195
115 -@lru_cache(maxsize=None)
116 -def _mm_token_str(model_config: ModelConfig, tokenizer: AnyTokenizer,
117 - modality: Literal["image", "audio"]) -> Optional[str]:
118 - # TODO: Let user specify how to insert image tokens into prompt
119 - # (similar to chat template)
120 - model_type = model_config.hf_config.model_type
121 - if modality == "image":
122 - if model_type == "phi3_v":
123 - # Workaround since this token is not defined in the tokenizer
124 - return "<|image_1|>"
125 - if model_type == "minicpmv":
126 - return "(<image>./</image>)"
127 - if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
128 - # These models do not use image tokens in the prompt
129 - return None
130 - if model_type.startswith("llava"):
131 - return tokenizer.decode(model_config.hf_config.image_token_index)
132 - if model_type in ("chameleon", "internvl_chat"):
133 - return "<image>"
134 -
135 - raise TypeError(f"Unknown model type: {model_type}")
136 - elif modality == "audio":
137 - if model_type == "ultravox":
138 - return "<|reserved_special_token_0|>"
139 - raise TypeError(f"Unknown model type: {model_type}")
140 - else:
141 - raise TypeError(f"Unknown modality: {modality}")
142 -
143 -
196 # TODO: Let user specify how to insert multimodal tokens into prompt
197 # (similar to chat template)
146 -def _get_full_multimodal_text_prompt(placeholder_token_str: str,
198 +def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
199 text_prompt: str) -> str:
200 """Combine multimodal prompts for a multimodal language model"""
201
150 - # NOTE: For now we assume all model architectures use the same
151 - # placeholder + text prompt format. This may change in the future.
152 - return f"{placeholder_token_str}\n{text_prompt}"
202 + # Look through the text prompt to check for missing placeholders
203 + missing_placeholders = []
204 + for placeholder in placeholder_counts:
205 +
206 + # For any existing placeholder in the text prompt, we leave it as is
207 + placeholder_counts[placeholder] -= text_prompt.count(placeholder)
208 +
209 + if placeholder_counts[placeholder] < 0:
210 + raise ValueError(
211 + f"Found more '{placeholder}' placeholders in input prompt than "
212 + "actual multimodal data items.")
213 +
214 + missing_placeholders.extend([placeholder] *
215 + placeholder_counts[placeholder])
216 +
217 + # NOTE: For now we always add missing placeholders at the front of
218 + # the prompt. This may change to be customizable in the future.
219 + return "\n".join(missing_placeholders + [text_prompt])
220
221
222 _TextParser = TypeAdapter(ChatCompletionContentPartTextParam)
@@ -160,12 +227,12 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
227 def _parse_chat_message_content_parts(
228 role: str,
229 parts: Iterable[ChatCompletionContentPartParam],
163 - model_config: ModelConfig,
164 - tokenizer: AnyTokenizer,
165 -) -> ChatMessageParseResult:
230 + mm_tracker: MultiModalItemTracker,
231 +) -> List[ConversationMessage]:
232 texts: List[str] = []
167 - mm_futures: List[Awaitable[MultiModalDataDict]] = []
168 - modality: Literal["image", "audio"] = "image"
233 +
234 + # multimodal placeholder_string : count
235 + mm_placeholder_counts: Dict[str, int] = {}
236
237 for part in parts:
238 part_type = part["type"]
@@ -173,11 +240,6 @@ def _parse_chat_message_content_parts(
240 text = _TextParser.validate_python(part)["text"]
241 texts.append(text)
242 elif part_type == "image_url":
176 - modality = "image"
177 - if len(mm_futures) > 0:
178 - raise NotImplementedError(
179 - "Multiple multimodal inputs is currently not supported.")
180 -
243 image_url = _ImageParser.validate_python(part)["image_url"]
244
245 if image_url.get("detail", "auto") != "auto":
@@ -185,60 +247,44 @@ def _parse_chat_message_content_parts(
247 "'image_url.detail' is currently not supported and "
248 "will be ignored.")
249
188 - image_future = async_get_and_parse_image(image_url["url"])
189 - mm_futures.append(image_future)
250 + image_coro = async_get_and_parse_image(image_url["url"])
251 + placeholder = mm_tracker.add("image", image_coro)
252 + if placeholder:
253 + mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
254 + placeholder, 0) + 1
255 elif part_type == "audio_url":
191 - modality = "audio"
192 - if len(mm_futures) > 0:
193 - raise NotImplementedError(
194 - "Multiple multimodal inputs is currently not supported.")
195 -
256 audio_url = _AudioParser.validate_python(part)["audio_url"]
197 - audio_future = async_get_and_parse_audio(audio_url["url"])
198 - mm_futures.append(audio_future)
257 + audio_coro = async_get_and_parse_audio(audio_url["url"])
258 + placeholder = mm_tracker.add("audio", audio_coro)
259 + if placeholder:
260 + mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
261 + placeholder, 0) + 1
262 else:
263 raise NotImplementedError(f"Unknown part type: {part_type}")
264
265 text_prompt = "\n".join(texts)
266 + if mm_placeholder_counts:
267 + text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
268 + text_prompt)
269
204 - if mm_futures:
205 - placeholder_token_str = _mm_token_str(model_config, tokenizer,
206 - modality)
207 - if placeholder_token_str is not None:
208 - if placeholder_token_str in text_prompt:
209 - logger.warning(
210 - "Detected multi-modal token string in the text prompt. "
211 - "Skipping prompt formatting.")
212 - else:
213 - text_prompt = _get_full_multimodal_text_prompt(
214 - placeholder_token_str=placeholder_token_str,
215 - text_prompt=text_prompt,
216 - )
217 -
218 - messages = [ConversationMessage(role=role, content=text_prompt)]
219 -
220 - return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
270 + return [ConversationMessage(role=role, content=text_prompt)]
271
272
273 def _parse_chat_message_content(
224 - message: ChatCompletionMessageParam,
225 - model_config: ModelConfig,
226 - tokenizer: AnyTokenizer,
227 -) -> ChatMessageParseResult:
274 + message: ChatCompletionMessageParam,
275 + mm_tracker: MultiModalItemTracker) -> List[ConversationMessage]:
276 role = message["role"]
277 content = message.get("content")
278
279 if content is None:
232 - return ChatMessageParseResult(messages=[], mm_futures=[])
280 + return []
281 if isinstance(content, str):
234 - messages = [ConversationMessage(role=role, content=content)]
235 - return ChatMessageParseResult(messages=messages, mm_futures=[])
282 + return [ConversationMessage(role=role, content=content)]
283
284 return _parse_chat_message_content_parts(
285 role,
286 content, # type: ignore
240 - model_config,
241 - tokenizer,
287 + mm_tracker,
288 )
289
290
@@ -246,18 +292,16 @@ def parse_chat_messages(
292 messages: List[ChatCompletionMessageParam],
293 model_config: ModelConfig,
294 tokenizer: AnyTokenizer,
249 -) -> Tuple[List[ConversationMessage], List[Awaitable[MultiModalDataDict]]]:
295 +) -> Tuple[List[ConversationMessage], Optional[Awaitable[MultiModalDataDict]]]:
296 conversation: List[ConversationMessage] = []
251 - mm_futures: List[Awaitable[MultiModalDataDict]] = []
297 + mm_tracker = MultiModalItemTracker(model_config, tokenizer)
298
299 for msg in messages:
254 - parse_result = _parse_chat_message_content(msg, model_config,
255 - tokenizer)
300 + sub_messages = _parse_chat_message_content(msg, mm_tracker)
301
257 - conversation.extend(parse_result.messages)
258 - mm_futures.extend(parse_result.mm_futures)
302 + conversation.extend(sub_messages)
303
260 - return conversation, mm_futures
304 + return conversation, mm_tracker.all_mm_data()
305
306
307 def apply_chat_template(
vllm/entrypoints/openai/serving_chat.py
+3 -7
@@ -94,7 +94,7 @@ class OpenAIServingChat(OpenAIServing):
94 tokenizer = await self.async_engine_client.get_tokenizer(
95 lora_request)
96
97 - conversation, mm_futures = parse_chat_messages(
97 + conversation, mm_data_future = parse_chat_messages(
98 request.messages, model_config, tokenizer)
99
100 tool_dicts = None if request.tools is None else [
@@ -116,12 +116,8 @@ class OpenAIServingChat(OpenAIServing):
116
117 mm_data: Optional[MultiModalDataDict] = None
118 try:
119 - if len(mm_futures):
120 - # since we support only single mm data currently
121 - assert len(
122 - mm_futures
123 - ) == 1, "Multiple 'image_url' input is currently not supported."
124 - mm_data = await mm_futures[0]
119 + if mm_data_future:
120 + mm_data = await mm_data_future
121 except Exception as e:
122 logger.error("Error in loading multi-modal data: %s", e)
123 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+2 -2
@@ -65,10 +65,10 @@ class OpenAIServingTokenization(OpenAIServing):
65 if isinstance(request, TokenizeChatRequest):
66 model_config = self.model_config
67
68 - conversation, mm_futures = parse_chat_messages(
68 + conversation, mm_data_future = parse_chat_messages(
69 request.messages, model_config, tokenizer)
70
71 - if mm_futures:
71 + if mm_data_future:
72 logger.warning(
73 "Multi-modal inputs are ignored during tokenization")
74