A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Multimodal support in offline chat (#8098)
Cyrus Leung committed
Sep 4, 2024 at 13:22 UTC
855c262a6bcbb392a6e312caa3489648aa3f4a47
8 files changed
+356
-112
tests/entrypoints/llm/test_generate.py
+34
@@ -6,6 +6,7 @@ import pytest
6
from vllm import LLM, RequestOutput, SamplingParams
7
8
from ...conftest import cleanup
9
+from ..openai.test_vision import TEST_IMAGE_URLS
10
11
MODEL_NAME = "facebook/opt-125m"
12
@@ -159,3 +160,36 @@ def test_chat():
160
]
161
outputs = llm.chat(messages)
162
assert len(outputs) == 1
163
+
164
+
165
+@pytest.mark.parametrize("image_urls",
166
+ [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
167
+def test_chat_multi_image(image_urls: List[str]):
168
+ llm = LLM(
169
+ model="microsoft/Phi-3.5-vision-instruct",
170
+ dtype="bfloat16",
171
+ max_model_len=4096,
172
+ max_num_seqs=5,
173
+ enforce_eager=True,
174
+ trust_remote_code=True,
175
+ limit_mm_per_prompt={"image": 2},
176
+ )
177
+
178
+ messages = [{
179
+ "role":
180
+ "user",
181
+ "content": [
182
+ *({
183
+ "type": "image_url",
184
+ "image_url": {
185
+ "url": image_url
186
+ }
187
+ } for image_url in image_urls),
188
+ {
189
+ "type": "text",
190
+ "text": "What's in this image?"
191
+ },
192
+ ],
193
+ }]
194
+ outputs = llm.chat(messages)
195
+ assert len(outputs) >= 0
tests/entrypoints/test_chat_utils.py
+124
-40
@@ -1,11 +1,14 @@
1
import warnings
2
+from typing import Optional
3
4
import pytest
5
from PIL import Image
6
7
from vllm.assets.image import ImageAsset
8
from vllm.config import ModelConfig
8
-from vllm.entrypoints.chat_utils import parse_chat_messages
9
+from vllm.entrypoints.chat_utils import (parse_chat_messages,
10
+ parse_chat_messages_futures)
11
+from vllm.multimodal import MultiModalDataDict
12
from vllm.multimodal.utils import encode_image_base64
13
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
14
@@ -42,10 +45,28 @@ def image_url():
45
return f"data:image/jpeg;base64,{base64}"
46
47
45
-@pytest.mark.asyncio
46
-async def test_parse_chat_messages_with_image_url(phi3v_model_config,
47
- phi3v_tokenizer, image_url):
48
- conversation, mm_future = parse_chat_messages([{
48
+def _assert_mm_data_is_image_input(
49
+ mm_data: Optional[MultiModalDataDict],
50
+ image_count: int,
51
+) -> None:
52
+ assert mm_data is not None
53
+ assert set(mm_data.keys()) == {"image"}
54
+
55
+ image_data = mm_data.get("image")
56
+ assert image_data is not None
57
+
58
+ if image_count == 1:
59
+ assert isinstance(image_data, Image.Image)
60
+ else:
61
+ assert isinstance(image_data, list) and len(image_data) == image_count
62
+
63
+
64
+def test_parse_chat_messages_single_image(
65
+ phi3v_model_config,
66
+ phi3v_tokenizer,
67
+ image_url,
68
+):
69
+ conversation, mm_data = parse_chat_messages([{
70
"role":
71
"user",
72
"content": [{
@@ -63,15 +84,42 @@ async def test_parse_chat_messages_with_image_url(phi3v_model_config,
84
"role": "user",
85
"content": "<|image_1|>\nWhat's in the image?"
86
}]
66
- mm_data = await mm_future
67
- assert set(mm_data.keys()) == {"image"}
68
- assert isinstance(mm_data["image"], Image.Image)
87
+ _assert_mm_data_is_image_input(mm_data, 1)
88
89
90
@pytest.mark.asyncio
72
-async def test_parse_chat_messages_multiple_images(phi3v_model_config,
73
- phi3v_tokenizer, image_url):
74
- conversation, mm_future = parse_chat_messages([{
91
+async def test_parse_chat_messages_single_image_async(
92
+ phi3v_model_config,
93
+ phi3v_tokenizer,
94
+ image_url,
95
+):
96
+ conversation, mm_future = parse_chat_messages_futures([{
97
+ "role":
98
+ "user",
99
+ "content": [{
100
+ "type": "image_url",
101
+ "image_url": {
102
+ "url": image_url
103
+ }
104
+ }, {
105
+ "type": "text",
106
+ "text": "What's in the image?"
107
+ }]
108
+ }], phi3v_model_config, phi3v_tokenizer)
109
+
110
+ assert conversation == [{
111
+ "role": "user",
112
+ "content": "<|image_1|>\nWhat's in the image?"
113
+ }]
114
+ _assert_mm_data_is_image_input(await mm_future, 1)
115
+
116
+
117
+def test_parse_chat_messages_multiple_images(
118
+ phi3v_model_config,
119
+ phi3v_tokenizer,
120
+ image_url,
121
+):
122
+ conversation, mm_data = parse_chat_messages([{
123
"role":
124
"user",
125
"content": [{
@@ -96,15 +144,49 @@ async def test_parse_chat_messages_multiple_images(phi3v_model_config,
144
"content":
145
"<|image_1|>\n<|image_2|>\nWhat's in these images?"
146
}]
99
- mm_data = await mm_future
100
- assert set(mm_data.keys()) == {"image"}
101
- assert len(mm_data["image"]) == 2
147
+ _assert_mm_data_is_image_input(mm_data, 2)
148
149
150
@pytest.mark.asyncio
105
-async def test_parse_chat_messages_placeholder_already_in_prompt(
106
- phi3v_model_config, phi3v_tokenizer, image_url):
107
- conversation, mm_future = parse_chat_messages([{
151
+async def test_parse_chat_messages_multiple_images_async(
152
+ phi3v_model_config,
153
+ phi3v_tokenizer,
154
+ image_url,
155
+):
156
+ conversation, mm_future = parse_chat_messages_futures([{
157
+ "role":
158
+ "user",
159
+ "content": [{
160
+ "type": "image_url",
161
+ "image_url": {
162
+ "url": image_url
163
+ }
164
+ }, {
165
+ "type": "image_url",
166
+ "image_url": {
167
+ "url": image_url
168
+ }
169
+ }, {
170
+ "type": "text",
171
+ "text": "What's in these images?"
172
+ }]
173
+ }], phi3v_model_config, phi3v_tokenizer)
174
+
175
+ assert conversation == [{
176
+ "role":
177
+ "user",
178
+ "content":
179
+ "<|image_1|>\n<|image_2|>\nWhat's in these images?"
180
+ }]
181
+ _assert_mm_data_is_image_input(await mm_future, 2)
182
+
183
+
184
+def test_parse_chat_messages_placeholder_already_in_prompt(
185
+ phi3v_model_config,
186
+ phi3v_tokenizer,
187
+ image_url,
188
+):
189
+ conversation, mm_data = parse_chat_messages([{
190
"role":
191
"user",
192
"content": [{
@@ -131,15 +213,15 @@ async def test_parse_chat_messages_placeholder_already_in_prompt(
213
"content":
214
"What's in <|image_1|> and how does it compare to <|image_2|>?"
215
}]
134
- mm_data = await mm_future
135
- assert set(mm_data.keys()) == {"image"}
136
- assert len(mm_data["image"]) == 2
216
+ _assert_mm_data_is_image_input(mm_data, 2)
217
218
139
-@pytest.mark.asyncio
140
-async def test_parse_chat_messages_placeholder_one_already_in_prompt(
141
- phi3v_model_config, phi3v_tokenizer, image_url):
142
- conversation, mm_future = parse_chat_messages([{
219
+def test_parse_chat_messages_placeholder_one_already_in_prompt(
220
+ phi3v_model_config,
221
+ phi3v_tokenizer,
222
+ image_url,
223
+):
224
+ conversation, mm_data = parse_chat_messages([{
225
"role":
226
"user",
227
"content": [{
@@ -167,15 +249,15 @@ async def test_parse_chat_messages_placeholder_one_already_in_prompt(
249
"<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "
250
"other one?"
251
}]
170
- mm_data = await mm_future
171
- assert set(mm_data.keys()) == {"image"}
172
- assert len(mm_data["image"]) == 2
252
+ _assert_mm_data_is_image_input(mm_data, 2)
253
254
175
-@pytest.mark.asyncio
176
-async def test_parse_chat_messages_multiple_images_across_messages(
177
- phi3v_model_config, phi3v_tokenizer, image_url):
178
- conversation, mm_future = parse_chat_messages([{
255
+def test_parse_chat_messages_multiple_images_across_messages(
256
+ phi3v_model_config,
257
+ phi3v_tokenizer,
258
+ image_url,
259
+):
260
+ conversation, mm_data = parse_chat_messages([{
261
"role":
262
"user",
263
"content": [{
@@ -218,14 +300,14 @@ async def test_parse_chat_messages_multiple_images_across_messages(
300
"content": "<|image_2|>\nWhat about this one?"
301
},
302
]
221
- mm_data = await mm_future
222
- assert set(mm_data.keys()) == {"image"}
223
- assert len(mm_data["image"]) == 2
303
+ _assert_mm_data_is_image_input(mm_data, 2)
304
305
226
-@pytest.mark.asyncio
227
-async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
228
- phi3v_model_config, phi3v_tokenizer, image_url):
306
+def test_parse_chat_messages_rejects_too_many_images_in_one_message(
307
+ phi3v_model_config,
308
+ phi3v_tokenizer,
309
+ image_url,
310
+):
311
with warnings.catch_warnings():
312
warnings.filterwarnings(
313
"ignore",
@@ -259,9 +341,11 @@ async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
341
}], phi3v_model_config, phi3v_tokenizer)
342
343
262
-@pytest.mark.asyncio
263
-async def test_parse_chat_messages_rejects_too_many_images_across_messages(
264
- phi3v_model_config, phi3v_tokenizer, image_url):
344
+def test_parse_chat_messages_rejects_too_many_images_across_messages(
345
+ phi3v_model_config,
346
+ phi3v_tokenizer,
347
+ image_url,
348
+):
349
with warnings.catch_warnings():
350
warnings.filterwarnings(
351
"ignore",
vllm/entrypoints/chat_utils.py
+159
-49
@@ -1,10 +1,11 @@
1
import asyncio
2
import codecs
3
+from abc import ABC, abstractmethod
4
from collections import defaultdict
5
from functools import lru_cache
6
from pathlib import Path
6
-from typing import (Any, Awaitable, Dict, Iterable, List, Literal, Mapping,
7
- Optional, Tuple, Union)
7
+from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal,
8
+ Mapping, Optional, Tuple, TypeVar, Union)
9
10
# yapf conflicts with isort for this block
11
# yapf: disable
@@ -23,7 +24,8 @@ from vllm.config import ModelConfig
24
from vllm.logger import init_logger
25
from vllm.multimodal import MultiModalDataDict
26
from vllm.multimodal.utils import (async_get_and_parse_audio,
26
- async_get_and_parse_image)
27
+ async_get_and_parse_image,
28
+ get_and_parse_audio, get_and_parse_image)
29
from vllm.transformers_utils.tokenizer import AnyTokenizer
30
31
logger = init_logger(__name__)
@@ -81,7 +83,11 @@ class ConversationMessage(TypedDict):
83
content: str
84
85
84
-class MultiModalItemTracker:
86
+ModalityStr = Literal["image", "audio"]
87
+_T = TypeVar("_T")
88
+
89
+
90
+class BaseMultiModalItemTracker(ABC, Generic[_T]):
91
"""
92
Tracks multi-modal items in a given request and ensures that the number
93
of multi-modal items in a given request does not exceed the configured
@@ -89,37 +95,28 @@ class MultiModalItemTracker:
95
"""
96
97
def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
98
+ super().__init__()
99
+
100
self._model_config = model_config
101
self._tokenizer = tokenizer
102
self._allowed_items = (model_config.multimodal_config.limit_per_prompt
103
if model_config.multimodal_config else {})
104
self._consumed_items = {k: 0 for k in self._allowed_items}
97
- self._futures: List[Awaitable[MultiModalDataDict]] = []
105
+
106
+ self._items: List[_T] = []
107
108
@staticmethod
109
@lru_cache(maxsize=None)
101
- def _cached_token_str(tokenizer: AnyTokenizer, token_index: int):
110
+ def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
111
return tokenizer.decode(token_index)
112
104
- def add(self, modality: Literal["image", "audio"],
105
- mm_future: Awaitable[MultiModalDataDict]) -> Optional[str]:
106
- """
107
- Adds the multi-modal item to the current prompt and returns the
108
- placeholder string to use, if any.
109
- """
110
- allowed_count = self._allowed_items.get(modality, 1)
111
- current_count = self._consumed_items.get(modality, 0) + 1
112
- if current_count > allowed_count:
113
- raise ValueError(
114
- f"At most {allowed_count} {modality}(s) may be provided in "
115
- "one request.")
116
-
117
- self._consumed_items[modality] = current_count
118
- self._futures.append(mm_future)
119
-
113
+ def _placeholder_str(self, modality: ModalityStr,
114
+ current_count: int) -> Optional[str]:
115
# TODO: Let user specify how to insert image tokens into prompt
116
# (similar to chat template)
122
- model_type = self._model_config.hf_config.model_type
117
+ hf_config = self._model_config.hf_config
118
+ model_type = hf_config.model_type
119
+
120
if modality == "image":
121
if model_type == "phi3_v":
122
# Workaround since this token is not defined in the tokenizer
@@ -130,9 +127,8 @@ class MultiModalItemTracker:
127
# These models do not use image tokens in the prompt
128
return None
129
if model_type.startswith("llava"):
133
- return MultiModalItemTracker._cached_token_str(
134
- self._tokenizer,
135
- self._model_config.hf_config.image_token_index)
130
+ return self._cached_token_str(self._tokenizer,
131
+ hf_config.image_token_index)
132
if model_type in ("chameleon", "internvl_chat"):
133
return "<image>"
134
@@ -145,11 +141,11 @@ class MultiModalItemTracker:
141
raise TypeError(f"Unknown modality: {modality}")
142
143
@staticmethod
148
- async def _combine(futures: List[Awaitable[MultiModalDataDict]]):
144
+ def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict:
145
mm_lists: Mapping[str, List[object]] = defaultdict(list)
146
147
# Merge all the multi-modal items
152
- for single_mm_data in (await asyncio.gather(*futures)):
148
+ for single_mm_data in items:
149
for mm_key, mm_item in single_mm_data.items():
150
if isinstance(mm_item, list):
151
mm_lists[mm_key].extend(mm_item)
@@ -162,9 +158,113 @@ class MultiModalItemTracker:
158
for mm_key, mm_list in mm_lists.items()
159
}
160
165
- def all_mm_data(self) -> Optional[Awaitable[MultiModalDataDict]]:
166
- return MultiModalItemTracker._combine(
167
- self._futures) if self._futures else None
161
+ def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
162
+ """
163
+ Add a multi-modal item to the current prompt and returns the
164
+ placeholder string to use, if any.
165
+ """
166
+ allowed_count = self._allowed_items.get(modality, 1)
167
+ current_count = self._consumed_items.get(modality, 0) + 1
168
+ if current_count > allowed_count:
169
+ raise ValueError(
170
+ f"At most {allowed_count} {modality}(s) may be provided in "
171
+ "one request.")
172
+
173
+ self._consumed_items[modality] = current_count
174
+ self._items.append(item)
175
+
176
+ return self._placeholder_str(modality, current_count)
177
+
178
+ @abstractmethod
179
+ def create_parser(self) -> "BaseMultiModalContentParser":
180
+ raise NotImplementedError
181
+
182
+
183
+class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]):
184
+
185
+ def all_mm_data(self) -> Optional[MultiModalDataDict]:
186
+ return self._combine(self._items) if self._items else None
187
+
188
+ def create_parser(self) -> "BaseMultiModalContentParser":
189
+ return MultiModalContentParser(self)
190
+
191
+
192
+class AsyncMultiModalItemTracker(
193
+ BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]):
194
+
195
+ async def all_mm_data(self) -> Optional[MultiModalDataDict]:
196
+ if self._items:
197
+ items = await asyncio.gather(*self._items)
198
+ return self._combine(items)
199
+
200
+ return None
201
+
202
+ def create_parser(self) -> "BaseMultiModalContentParser":
203
+ return AsyncMultiModalContentParser(self)
204
+
205
+
206
+class BaseMultiModalContentParser(ABC):
207
+
208
+ def __init__(self) -> None:
209
+ super().__init__()
210
+
211
+ # multimodal placeholder_string : count
212
+ self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0)
213
+
214
+ def _add_placeholder(self, placeholder: Optional[str]):
215
+ if placeholder:
216
+ self._placeholder_counts[placeholder] += 1
217
+
218
+ def mm_placeholder_counts(self) -> Dict[str, int]:
219
+ return dict(self._placeholder_counts)
220
+
221
+ @abstractmethod
222
+ def parse_image(self, image_url: str) -> None:
223
+ raise NotImplementedError
224
+
225
+ @abstractmethod
226
+ def parse_audio(self, audio_url: str) -> None:
227
+ raise NotImplementedError
228
+
229
+
230
+class MultiModalContentParser(BaseMultiModalContentParser):
231
+
232
+ def __init__(self, tracker: MultiModalItemTracker) -> None:
233
+ super().__init__()
234
+
235
+ self._tracker = tracker
236
+
237
+ def parse_image(self, image_url: str) -> None:
238
+ image = get_and_parse_image(image_url)
239
+
240
+ placeholder = self._tracker.add("image", image)
241
+ self._add_placeholder(placeholder)
242
+
243
+ def parse_audio(self, audio_url: str) -> None:
244
+ audio = get_and_parse_audio(audio_url)
245
+
246
+ placeholder = self._tracker.add("audio", audio)
247
+ self._add_placeholder(placeholder)
248
+
249
+
250
+class AsyncMultiModalContentParser(BaseMultiModalContentParser):
251
+
252
+ def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
253
+ super().__init__()
254
+
255
+ self._tracker = tracker
256
+
257
+ def parse_image(self, image_url: str) -> None:
258
+ image_coro = async_get_and_parse_image(image_url)
259
+
260
+ placeholder = self._tracker.add("image", image_coro)
261
+ self._add_placeholder(placeholder)
262
+
263
+ def parse_audio(self, audio_url: str) -> None:
264
+ audio_coro = async_get_and_parse_audio(audio_url)
265
+
266
+ placeholder = self._tracker.add("audio", audio_coro)
267
+ self._add_placeholder(placeholder)
268
269
270
def load_chat_template(
@@ -197,10 +297,10 @@ def load_chat_template(
297
# (similar to chat template)
298
def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
299
text_prompt: str) -> str:
200
- """Combine multimodal prompts for a multimodal language model"""
300
+ """Combine multimodal prompts for a multimodal language model."""
301
302
# Look through the text prompt to check for missing placeholders
203
- missing_placeholders = []
303
+ missing_placeholders: List[str] = []
304
for placeholder in placeholder_counts:
305
306
# For any existing placeholder in the text prompt, we leave it as is
@@ -227,12 +327,11 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
327
def _parse_chat_message_content_parts(
328
role: str,
329
parts: Iterable[ChatCompletionContentPartParam],
230
- mm_tracker: MultiModalItemTracker,
330
+ mm_tracker: BaseMultiModalItemTracker,
331
) -> List[ConversationMessage]:
332
texts: List[str] = []
333
234
- # multimodal placeholder_string : count
235
- mm_placeholder_counts: Dict[str, int] = {}
334
+ mm_parser = mm_tracker.create_parser()
335
336
for part in parts:
337
part_type = part["type"]
@@ -247,22 +346,16 @@ def _parse_chat_message_content_parts(
346
"'image_url.detail' is currently not supported and "
347
"will be ignored.")
348
250
- image_coro = async_get_and_parse_image(image_url["url"])
251
- placeholder = mm_tracker.add("image", image_coro)
252
- if placeholder:
253
- mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
254
- placeholder, 0) + 1
349
+ mm_parser.parse_image(image_url["url"])
350
elif part_type == "audio_url":
351
audio_url = _AudioParser.validate_python(part)["audio_url"]
257
- audio_coro = async_get_and_parse_audio(audio_url["url"])
258
- placeholder = mm_tracker.add("audio", audio_coro)
259
- if placeholder:
260
- mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
261
- placeholder, 0) + 1
352
+
353
+ mm_parser.parse_audio(audio_url["url"])
354
else:
355
raise NotImplementedError(f"Unknown part type: {part_type}")
356
357
text_prompt = "\n".join(texts)
358
+ mm_placeholder_counts = mm_parser.mm_placeholder_counts()
359
if mm_placeholder_counts:
360
text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
361
text_prompt)
@@ -271,8 +364,9 @@ def _parse_chat_message_content_parts(
364
365
366
def _parse_chat_message_content(
274
- message: ChatCompletionMessageParam,
275
- mm_tracker: MultiModalItemTracker) -> List[ConversationMessage]:
367
+ message: ChatCompletionMessageParam,
368
+ mm_tracker: BaseMultiModalItemTracker,
369
+) -> List[ConversationMessage]:
370
role = message["role"]
371
content = message.get("content")
372
@@ -292,7 +386,7 @@ def parse_chat_messages(
386
messages: List[ChatCompletionMessageParam],
387
model_config: ModelConfig,
388
tokenizer: AnyTokenizer,
295
-) -> Tuple[List[ConversationMessage], Optional[Awaitable[MultiModalDataDict]]]:
389
+) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:
390
conversation: List[ConversationMessage] = []
391
mm_tracker = MultiModalItemTracker(model_config, tokenizer)
392
@@ -304,6 +398,22 @@ def parse_chat_messages(
398
return conversation, mm_tracker.all_mm_data()
399
400
401
+def parse_chat_messages_futures(
402
+ messages: List[ChatCompletionMessageParam],
403
+ model_config: ModelConfig,
404
+ tokenizer: AnyTokenizer,
405
+) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
406
+ conversation: List[ConversationMessage] = []
407
+ mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
408
+
409
+ for msg in messages:
410
+ sub_messages = _parse_chat_message_content(msg, mm_tracker)
411
+
412
+ conversation.extend(sub_messages)
413
+
414
+ return conversation, mm_tracker.all_mm_data()
415
+
416
+
417
def apply_chat_template(
418
tokenizer: AnyTokenizer,
419
conversation: List[ConversationMessage],
vllm/entrypoints/llm.py
+19
-12
@@ -23,7 +23,7 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23
get_cached_tokenizer)
24
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
25
from vllm.usage.usage_lib import UsageContext
26
-from vllm.utils import Counter, deprecate_kwargs
26
+from vllm.utils import Counter, deprecate_kwargs, is_list_of
27
28
logger = init_logger(__name__)
29
@@ -358,15 +358,18 @@ class LLM:
358
add_generation_prompt: bool = True,
359
) -> List[RequestOutput]:
360
"""
361
- Generates responses for chat messages.
361
+ Generate responses for a chat conversation.
362
363
- Converts the messages to prompts using the tokenizer and calls
364
- the :meth:`generate` method to generate the responses.
363
+ The chat conversation is converted into a text prompt using the
364
+ tokenizer and calls the :meth:`generate` method to generate the
365
+ responses.
366
+
367
+ Multi-modal inputs can be passed in the same way you would pass them
368
+ to the OpenAI API.
369
370
Args:
367
- messages: A list of messages to generate responses for. Each
368
- message is a list of dictionaries with 'role' and 'content'
369
- keys.
371
+ messages: A single conversation represented as a list of messages.
372
+ Each message is a dictionary with 'role' and 'content' keys.
373
sampling_params: The sampling parameters for text generation.
374
If None, we use the default sampling parameters. When it
375
is a single value, it is applied to every prompt. When it
@@ -387,21 +390,25 @@ class LLM:
390
tokenizer = self.get_tokenizer()
391
model_config = self.llm_engine.get_model_config()
392
390
- conversations, _ = parse_chat_messages(messages, model_config,
391
- tokenizer)
393
+ conversation, mm_data = parse_chat_messages(messages, model_config,
394
+ tokenizer)
395
396
prompt = apply_chat_template(
397
tokenizer,
395
- conversations,
398
+ conversation,
399
chat_template=chat_template,
397
- add_generation_prompt=add_generation_prompt)
400
+ add_generation_prompt=add_generation_prompt,
401
+ )
402
403
inputs: PromptInputs
400
- if isinstance(prompt, list) and isinstance(prompt[0], int):
404
+ if is_list_of(prompt, int):
405
inputs = TokensPrompt(prompt_token_ids=prompt)
406
else:
407
inputs = TextPrompt(prompt=prompt)
408
409
+ if mm_data is not None:
410
+ inputs["multi_modal_data"] = mm_data
411
+
412
return self.generate(
413
inputs,
414
sampling_params=sampling_params,
vllm/entrypoints/openai/serving_chat.py
+3
-6
@@ -11,7 +11,7 @@ from vllm.engine.protocol import AsyncEngineClient
11
from vllm.entrypoints.chat_utils import (ConversationMessage,
12
apply_chat_template,
13
load_chat_template,
14
- parse_chat_messages)
14
+ parse_chat_messages_futures)
15
from vllm.entrypoints.logger import RequestLogger
16
from vllm.entrypoints.openai.protocol import (
17
ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -26,7 +26,6 @@ from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
26
TextTokensPrompt)
27
from vllm.inputs import TokensPrompt
28
from vllm.logger import init_logger
29
-from vllm.multimodal import MultiModalDataDict
29
from vllm.outputs import RequestOutput
30
from vllm.sequence import Logprob
31
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -94,7 +93,7 @@ class OpenAIServingChat(OpenAIServing):
93
tokenizer = await self.async_engine_client.get_tokenizer(
94
lora_request)
95
97
- conversation, mm_data_future = parse_chat_messages(
96
+ conversation, mm_data_future = parse_chat_messages_futures(
97
request.messages, model_config, tokenizer)
98
99
tool_dicts = None if request.tools is None else [
@@ -114,10 +113,8 @@ class OpenAIServingChat(OpenAIServing):
113
logger.error("Error in applying chat template from request: %s", e)
114
return self.create_error_response(str(e))
115
117
- mm_data: Optional[MultiModalDataDict] = None
116
try:
119
- if mm_data_future:
120
- mm_data = await mm_data_future
117
+ mm_data = await mm_data_future
118
except Exception as e:
119
logger.error("Error in loading multi-modal data: %s", e)
120
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+4
-3
@@ -4,7 +4,7 @@ from vllm.config import ModelConfig
4
from vllm.engine.protocol import AsyncEngineClient
5
from vllm.entrypoints.chat_utils import (apply_chat_template,
6
load_chat_template,
7
- parse_chat_messages)
7
+ parse_chat_messages_futures)
8
from vllm.entrypoints.logger import RequestLogger
9
# yapf conflicts with isort for this block
10
# yapf: disable
@@ -65,10 +65,11 @@ class OpenAIServingTokenization(OpenAIServing):
65
if isinstance(request, TokenizeChatRequest):
66
model_config = self.model_config
67
68
- conversation, mm_data_future = parse_chat_messages(
68
+ conversation, mm_data_future = parse_chat_messages_futures(
69
request.messages, model_config, tokenizer)
70
71
- if mm_data_future:
71
+ mm_data = await mm_data_future
72
+ if mm_data:
73
logger.warning(
74
"Multi-modal inputs are ignored during tokenization")
75
vllm/multimodal/utils.py
+10
@@ -120,6 +120,16 @@ async def async_fetch_audio(
120
return librosa.load(BytesIO(audio_bytes), sr=None)
121
122
123
+def get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
124
+ audio, sr = fetch_audio(audio_url)
125
+ return {"audio": (audio, sr)}
126
+
127
+
128
+def get_and_parse_image(image_url: str) -> MultiModalDataDict:
129
+ image = fetch_image(image_url)
130
+ return {"image": image}
131
+
132
+
133
async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
134
audio, sr = await async_fetch_audio(audio_url)
135
return {"audio": (audio, sr)}
vllm/transformers_utils/tokenizers/mistral.py
+3
-2
@@ -52,12 +52,13 @@ class MistralTokenizer:
52
assert isinstance(self.tokenizer,
53
(Tekkenizer, SentencePieceTokenizer)), type(
54
self.tokenizer)
55
- self._is_tekken = isinstance(self.tokenizer, Tekkenizer)
55
57
- if self._is_tekken:
56
+ if (is_tekken := isinstance(self.tokenizer, Tekkenizer)):
57
# Make sure special tokens will not raise
58
self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
59
60
+ self._is_tekken = is_tekken
61
+
62
# the following attributes are set to fit VLLM's design
63
self.is_fast = True
64
self.chat_template = True