@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 855c262a6b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Multimodal support in offline chat (#8098)

Cyrus Leung committed Sep 4, 2024 at 13:22 UTC 855c262a6bcbb392a6e312caa3489648aa3f4a47
8 files changed +356 -112
tests/entrypoints/llm/test_generate.py
+34
@@ -6,6 +6,7 @@ import pytest
6 from vllm import LLM, RequestOutput, SamplingParams
7
8 from ...conftest import cleanup
9 +from ..openai.test_vision import TEST_IMAGE_URLS
10
11 MODEL_NAME = "facebook/opt-125m"
12
@@ -159,3 +160,36 @@ def test_chat():
160 ]
161 outputs = llm.chat(messages)
162 assert len(outputs) == 1
163 +
164 +
165 +@pytest.mark.parametrize("image_urls",
166 + [[TEST_IMAGE_URLS[0], TEST_IMAGE_URLS[1]]])
167 +def test_chat_multi_image(image_urls: List[str]):
168 + llm = LLM(
169 + model="microsoft/Phi-3.5-vision-instruct",
170 + dtype="bfloat16",
171 + max_model_len=4096,
172 + max_num_seqs=5,
173 + enforce_eager=True,
174 + trust_remote_code=True,
175 + limit_mm_per_prompt={"image": 2},
176 + )
177 +
178 + messages = [{
179 + "role":
180 + "user",
181 + "content": [
182 + *({
183 + "type": "image_url",
184 + "image_url": {
185 + "url": image_url
186 + }
187 + } for image_url in image_urls),
188 + {
189 + "type": "text",
190 + "text": "What's in this image?"
191 + },
192 + ],
193 + }]
194 + outputs = llm.chat(messages)
195 + assert len(outputs) >= 0
tests/entrypoints/test_chat_utils.py
+124 -40
@@ -1,11 +1,14 @@
1 import warnings
2 +from typing import Optional
3
4 import pytest
5 from PIL import Image
6
7 from vllm.assets.image import ImageAsset
8 from vllm.config import ModelConfig
8 -from vllm.entrypoints.chat_utils import parse_chat_messages
9 +from vllm.entrypoints.chat_utils import (parse_chat_messages,
10 + parse_chat_messages_futures)
11 +from vllm.multimodal import MultiModalDataDict
12 from vllm.multimodal.utils import encode_image_base64
13 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
14
@@ -42,10 +45,28 @@ def image_url():
45 return f"data:image/jpeg;base64,{base64}"
46
47
45 -@pytest.mark.asyncio
46 -async def test_parse_chat_messages_with_image_url(phi3v_model_config,
47 - phi3v_tokenizer, image_url):
48 - conversation, mm_future = parse_chat_messages([{
48 +def _assert_mm_data_is_image_input(
49 + mm_data: Optional[MultiModalDataDict],
50 + image_count: int,
51 +) -> None:
52 + assert mm_data is not None
53 + assert set(mm_data.keys()) == {"image"}
54 +
55 + image_data = mm_data.get("image")
56 + assert image_data is not None
57 +
58 + if image_count == 1:
59 + assert isinstance(image_data, Image.Image)
60 + else:
61 + assert isinstance(image_data, list) and len(image_data) == image_count
62 +
63 +
64 +def test_parse_chat_messages_single_image(
65 + phi3v_model_config,
66 + phi3v_tokenizer,
67 + image_url,
68 +):
69 + conversation, mm_data = parse_chat_messages([{
70 "role":
71 "user",
72 "content": [{
@@ -63,15 +84,42 @@ async def test_parse_chat_messages_with_image_url(phi3v_model_config,
84 "role": "user",
85 "content": "<|image_1|>\nWhat's in the image?"
86 }]
66 - mm_data = await mm_future
67 - assert set(mm_data.keys()) == {"image"}
68 - assert isinstance(mm_data["image"], Image.Image)
87 + _assert_mm_data_is_image_input(mm_data, 1)
88
89
90 @pytest.mark.asyncio
72 -async def test_parse_chat_messages_multiple_images(phi3v_model_config,
73 - phi3v_tokenizer, image_url):
74 - conversation, mm_future = parse_chat_messages([{
91 +async def test_parse_chat_messages_single_image_async(
92 + phi3v_model_config,
93 + phi3v_tokenizer,
94 + image_url,
95 +):
96 + conversation, mm_future = parse_chat_messages_futures([{
97 + "role":
98 + "user",
99 + "content": [{
100 + "type": "image_url",
101 + "image_url": {
102 + "url": image_url
103 + }
104 + }, {
105 + "type": "text",
106 + "text": "What's in the image?"
107 + }]
108 + }], phi3v_model_config, phi3v_tokenizer)
109 +
110 + assert conversation == [{
111 + "role": "user",
112 + "content": "<|image_1|>\nWhat's in the image?"
113 + }]
114 + _assert_mm_data_is_image_input(await mm_future, 1)
115 +
116 +
117 +def test_parse_chat_messages_multiple_images(
118 + phi3v_model_config,
119 + phi3v_tokenizer,
120 + image_url,
121 +):
122 + conversation, mm_data = parse_chat_messages([{
123 "role":
124 "user",
125 "content": [{
@@ -96,15 +144,49 @@ async def test_parse_chat_messages_multiple_images(phi3v_model_config,
144 "content":
145 "<|image_1|>\n<|image_2|>\nWhat's in these images?"
146 }]
99 - mm_data = await mm_future
100 - assert set(mm_data.keys()) == {"image"}
101 - assert len(mm_data["image"]) == 2
147 + _assert_mm_data_is_image_input(mm_data, 2)
148
149
150 @pytest.mark.asyncio
105 -async def test_parse_chat_messages_placeholder_already_in_prompt(
106 - phi3v_model_config, phi3v_tokenizer, image_url):
107 - conversation, mm_future = parse_chat_messages([{
151 +async def test_parse_chat_messages_multiple_images_async(
152 + phi3v_model_config,
153 + phi3v_tokenizer,
154 + image_url,
155 +):
156 + conversation, mm_future = parse_chat_messages_futures([{
157 + "role":
158 + "user",
159 + "content": [{
160 + "type": "image_url",
161 + "image_url": {
162 + "url": image_url
163 + }
164 + }, {
165 + "type": "image_url",
166 + "image_url": {
167 + "url": image_url
168 + }
169 + }, {
170 + "type": "text",
171 + "text": "What's in these images?"
172 + }]
173 + }], phi3v_model_config, phi3v_tokenizer)
174 +
175 + assert conversation == [{
176 + "role":
177 + "user",
178 + "content":
179 + "<|image_1|>\n<|image_2|>\nWhat's in these images?"
180 + }]
181 + _assert_mm_data_is_image_input(await mm_future, 2)
182 +
183 +
184 +def test_parse_chat_messages_placeholder_already_in_prompt(
185 + phi3v_model_config,
186 + phi3v_tokenizer,
187 + image_url,
188 +):
189 + conversation, mm_data = parse_chat_messages([{
190 "role":
191 "user",
192 "content": [{
@@ -131,15 +213,15 @@ async def test_parse_chat_messages_placeholder_already_in_prompt(
213 "content":
214 "What's in <|image_1|> and how does it compare to <|image_2|>?"
215 }]
134 - mm_data = await mm_future
135 - assert set(mm_data.keys()) == {"image"}
136 - assert len(mm_data["image"]) == 2
216 + _assert_mm_data_is_image_input(mm_data, 2)
217
218
139 -@pytest.mark.asyncio
140 -async def test_parse_chat_messages_placeholder_one_already_in_prompt(
141 - phi3v_model_config, phi3v_tokenizer, image_url):
142 - conversation, mm_future = parse_chat_messages([{
219 +def test_parse_chat_messages_placeholder_one_already_in_prompt(
220 + phi3v_model_config,
221 + phi3v_tokenizer,
222 + image_url,
223 +):
224 + conversation, mm_data = parse_chat_messages([{
225 "role":
226 "user",
227 "content": [{
@@ -167,15 +249,15 @@ async def test_parse_chat_messages_placeholder_one_already_in_prompt(
249 "<|image_2|>\nWhat's in <|image_1|> and how does it compare to the "
250 "other one?"
251 }]
170 - mm_data = await mm_future
171 - assert set(mm_data.keys()) == {"image"}
172 - assert len(mm_data["image"]) == 2
252 + _assert_mm_data_is_image_input(mm_data, 2)
253
254
175 -@pytest.mark.asyncio
176 -async def test_parse_chat_messages_multiple_images_across_messages(
177 - phi3v_model_config, phi3v_tokenizer, image_url):
178 - conversation, mm_future = parse_chat_messages([{
255 +def test_parse_chat_messages_multiple_images_across_messages(
256 + phi3v_model_config,
257 + phi3v_tokenizer,
258 + image_url,
259 +):
260 + conversation, mm_data = parse_chat_messages([{
261 "role":
262 "user",
263 "content": [{
@@ -218,14 +300,14 @@ async def test_parse_chat_messages_multiple_images_across_messages(
300 "content": "<|image_2|>\nWhat about this one?"
301 },
302 ]
221 - mm_data = await mm_future
222 - assert set(mm_data.keys()) == {"image"}
223 - assert len(mm_data["image"]) == 2
303 + _assert_mm_data_is_image_input(mm_data, 2)
304
305
226 -@pytest.mark.asyncio
227 -async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
228 - phi3v_model_config, phi3v_tokenizer, image_url):
306 +def test_parse_chat_messages_rejects_too_many_images_in_one_message(
307 + phi3v_model_config,
308 + phi3v_tokenizer,
309 + image_url,
310 +):
311 with warnings.catch_warnings():
312 warnings.filterwarnings(
313 "ignore",
@@ -259,9 +341,11 @@ async def test_parse_chat_messages_rejects_too_many_images_in_one_message(
341 }], phi3v_model_config, phi3v_tokenizer)
342
343
262 -@pytest.mark.asyncio
263 -async def test_parse_chat_messages_rejects_too_many_images_across_messages(
264 - phi3v_model_config, phi3v_tokenizer, image_url):
344 +def test_parse_chat_messages_rejects_too_many_images_across_messages(
345 + phi3v_model_config,
346 + phi3v_tokenizer,
347 + image_url,
348 +):
349 with warnings.catch_warnings():
350 warnings.filterwarnings(
351 "ignore",
vllm/entrypoints/chat_utils.py
+159 -49
@@ -1,10 +1,11 @@
1 import asyncio
2 import codecs
3 +from abc import ABC, abstractmethod
4 from collections import defaultdict
5 from functools import lru_cache
6 from pathlib import Path
6 -from typing import (Any, Awaitable, Dict, Iterable, List, Literal, Mapping,
7 - Optional, Tuple, Union)
7 +from typing import (Any, Awaitable, Dict, Generic, Iterable, List, Literal,
8 + Mapping, Optional, Tuple, TypeVar, Union)
9
10 # yapf conflicts with isort for this block
11 # yapf: disable
@@ -23,7 +24,8 @@ from vllm.config import ModelConfig
24 from vllm.logger import init_logger
25 from vllm.multimodal import MultiModalDataDict
26 from vllm.multimodal.utils import (async_get_and_parse_audio,
26 - async_get_and_parse_image)
27 + async_get_and_parse_image,
28 + get_and_parse_audio, get_and_parse_image)
29 from vllm.transformers_utils.tokenizer import AnyTokenizer
30
31 logger = init_logger(__name__)
@@ -81,7 +83,11 @@ class ConversationMessage(TypedDict):
83 content: str
84
85
84 -class MultiModalItemTracker:
86 +ModalityStr = Literal["image", "audio"]
87 +_T = TypeVar("_T")
88 +
89 +
90 +class BaseMultiModalItemTracker(ABC, Generic[_T]):
91 """
92 Tracks multi-modal items in a given request and ensures that the number
93 of multi-modal items in a given request does not exceed the configured
@@ -89,37 +95,28 @@ class MultiModalItemTracker:
95 """
96
97 def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
98 + super().__init__()
99 +
100 self._model_config = model_config
101 self._tokenizer = tokenizer
102 self._allowed_items = (model_config.multimodal_config.limit_per_prompt
103 if model_config.multimodal_config else {})
104 self._consumed_items = {k: 0 for k in self._allowed_items}
97 - self._futures: List[Awaitable[MultiModalDataDict]] = []
105 +
106 + self._items: List[_T] = []
107
108 @staticmethod
109 @lru_cache(maxsize=None)
101 - def _cached_token_str(tokenizer: AnyTokenizer, token_index: int):
110 + def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
111 return tokenizer.decode(token_index)
112
104 - def add(self, modality: Literal["image", "audio"],
105 - mm_future: Awaitable[MultiModalDataDict]) -> Optional[str]:
106 - """
107 - Adds the multi-modal item to the current prompt and returns the
108 - placeholder string to use, if any.
109 - """
110 - allowed_count = self._allowed_items.get(modality, 1)
111 - current_count = self._consumed_items.get(modality, 0) + 1
112 - if current_count > allowed_count:
113 - raise ValueError(
114 - f"At most {allowed_count} {modality}(s) may be provided in "
115 - "one request.")
116 -
117 - self._consumed_items[modality] = current_count
118 - self._futures.append(mm_future)
119 -
113 + def _placeholder_str(self, modality: ModalityStr,
114 + current_count: int) -> Optional[str]:
115 # TODO: Let user specify how to insert image tokens into prompt
116 # (similar to chat template)
122 - model_type = self._model_config.hf_config.model_type
117 + hf_config = self._model_config.hf_config
118 + model_type = hf_config.model_type
119 +
120 if modality == "image":
121 if model_type == "phi3_v":
122 # Workaround since this token is not defined in the tokenizer
@@ -130,9 +127,8 @@ class MultiModalItemTracker:
127 # These models do not use image tokens in the prompt
128 return None
129 if model_type.startswith("llava"):
133 - return MultiModalItemTracker._cached_token_str(
134 - self._tokenizer,
135 - self._model_config.hf_config.image_token_index)
130 + return self._cached_token_str(self._tokenizer,
131 + hf_config.image_token_index)
132 if model_type in ("chameleon", "internvl_chat"):
133 return "<image>"
134
@@ -145,11 +141,11 @@ class MultiModalItemTracker:
141 raise TypeError(f"Unknown modality: {modality}")
142
143 @staticmethod
148 - async def _combine(futures: List[Awaitable[MultiModalDataDict]]):
144 + def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict:
145 mm_lists: Mapping[str, List[object]] = defaultdict(list)
146
147 # Merge all the multi-modal items
152 - for single_mm_data in (await asyncio.gather(*futures)):
148 + for single_mm_data in items:
149 for mm_key, mm_item in single_mm_data.items():
150 if isinstance(mm_item, list):
151 mm_lists[mm_key].extend(mm_item)
@@ -162,9 +158,113 @@ class MultiModalItemTracker:
158 for mm_key, mm_list in mm_lists.items()
159 }
160
165 - def all_mm_data(self) -> Optional[Awaitable[MultiModalDataDict]]:
166 - return MultiModalItemTracker._combine(
167 - self._futures) if self._futures else None
161 + def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
162 + """
163 + Add a multi-modal item to the current prompt and returns the
164 + placeholder string to use, if any.
165 + """
166 + allowed_count = self._allowed_items.get(modality, 1)
167 + current_count = self._consumed_items.get(modality, 0) + 1
168 + if current_count > allowed_count:
169 + raise ValueError(
170 + f"At most {allowed_count} {modality}(s) may be provided in "
171 + "one request.")
172 +
173 + self._consumed_items[modality] = current_count
174 + self._items.append(item)
175 +
176 + return self._placeholder_str(modality, current_count)
177 +
178 + @abstractmethod
179 + def create_parser(self) -> "BaseMultiModalContentParser":
180 + raise NotImplementedError
181 +
182 +
183 +class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]):
184 +
185 + def all_mm_data(self) -> Optional[MultiModalDataDict]:
186 + return self._combine(self._items) if self._items else None
187 +
188 + def create_parser(self) -> "BaseMultiModalContentParser":
189 + return MultiModalContentParser(self)
190 +
191 +
192 +class AsyncMultiModalItemTracker(
193 + BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]):
194 +
195 + async def all_mm_data(self) -> Optional[MultiModalDataDict]:
196 + if self._items:
197 + items = await asyncio.gather(*self._items)
198 + return self._combine(items)
199 +
200 + return None
201 +
202 + def create_parser(self) -> "BaseMultiModalContentParser":
203 + return AsyncMultiModalContentParser(self)
204 +
205 +
206 +class BaseMultiModalContentParser(ABC):
207 +
208 + def __init__(self) -> None:
209 + super().__init__()
210 +
211 + # multimodal placeholder_string : count
212 + self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0)
213 +
214 + def _add_placeholder(self, placeholder: Optional[str]):
215 + if placeholder:
216 + self._placeholder_counts[placeholder] += 1
217 +
218 + def mm_placeholder_counts(self) -> Dict[str, int]:
219 + return dict(self._placeholder_counts)
220 +
221 + @abstractmethod
222 + def parse_image(self, image_url: str) -> None:
223 + raise NotImplementedError
224 +
225 + @abstractmethod
226 + def parse_audio(self, audio_url: str) -> None:
227 + raise NotImplementedError
228 +
229 +
230 +class MultiModalContentParser(BaseMultiModalContentParser):
231 +
232 + def __init__(self, tracker: MultiModalItemTracker) -> None:
233 + super().__init__()
234 +
235 + self._tracker = tracker
236 +
237 + def parse_image(self, image_url: str) -> None:
238 + image = get_and_parse_image(image_url)
239 +
240 + placeholder = self._tracker.add("image", image)
241 + self._add_placeholder(placeholder)
242 +
243 + def parse_audio(self, audio_url: str) -> None:
244 + audio = get_and_parse_audio(audio_url)
245 +
246 + placeholder = self._tracker.add("audio", audio)
247 + self._add_placeholder(placeholder)
248 +
249 +
250 +class AsyncMultiModalContentParser(BaseMultiModalContentParser):
251 +
252 + def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
253 + super().__init__()
254 +
255 + self._tracker = tracker
256 +
257 + def parse_image(self, image_url: str) -> None:
258 + image_coro = async_get_and_parse_image(image_url)
259 +
260 + placeholder = self._tracker.add("image", image_coro)
261 + self._add_placeholder(placeholder)
262 +
263 + def parse_audio(self, audio_url: str) -> None:
264 + audio_coro = async_get_and_parse_audio(audio_url)
265 +
266 + placeholder = self._tracker.add("audio", audio_coro)
267 + self._add_placeholder(placeholder)
268
269
270 def load_chat_template(
@@ -197,10 +297,10 @@ def load_chat_template(
297 # (similar to chat template)
298 def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
299 text_prompt: str) -> str:
200 - """Combine multimodal prompts for a multimodal language model"""
300 + """Combine multimodal prompts for a multimodal language model."""
301
302 # Look through the text prompt to check for missing placeholders
203 - missing_placeholders = []
303 + missing_placeholders: List[str] = []
304 for placeholder in placeholder_counts:
305
306 # For any existing placeholder in the text prompt, we leave it as is
@@ -227,12 +327,11 @@ _AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam)
327 def _parse_chat_message_content_parts(
328 role: str,
329 parts: Iterable[ChatCompletionContentPartParam],
230 - mm_tracker: MultiModalItemTracker,
330 + mm_tracker: BaseMultiModalItemTracker,
331 ) -> List[ConversationMessage]:
332 texts: List[str] = []
333
234 - # multimodal placeholder_string : count
235 - mm_placeholder_counts: Dict[str, int] = {}
334 + mm_parser = mm_tracker.create_parser()
335
336 for part in parts:
337 part_type = part["type"]
@@ -247,22 +346,16 @@ def _parse_chat_message_content_parts(
346 "'image_url.detail' is currently not supported and "
347 "will be ignored.")
348
250 - image_coro = async_get_and_parse_image(image_url["url"])
251 - placeholder = mm_tracker.add("image", image_coro)
252 - if placeholder:
253 - mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
254 - placeholder, 0) + 1
349 + mm_parser.parse_image(image_url["url"])
350 elif part_type == "audio_url":
351 audio_url = _AudioParser.validate_python(part)["audio_url"]
257 - audio_coro = async_get_and_parse_audio(audio_url["url"])
258 - placeholder = mm_tracker.add("audio", audio_coro)
259 - if placeholder:
260 - mm_placeholder_counts[placeholder] = mm_placeholder_counts.get(
261 - placeholder, 0) + 1
352 +
353 + mm_parser.parse_audio(audio_url["url"])
354 else:
355 raise NotImplementedError(f"Unknown part type: {part_type}")
356
357 text_prompt = "\n".join(texts)
358 + mm_placeholder_counts = mm_parser.mm_placeholder_counts()
359 if mm_placeholder_counts:
360 text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
361 text_prompt)
@@ -271,8 +364,9 @@ def _parse_chat_message_content_parts(
364
365
366 def _parse_chat_message_content(
274 - message: ChatCompletionMessageParam,
275 - mm_tracker: MultiModalItemTracker) -> List[ConversationMessage]:
367 + message: ChatCompletionMessageParam,
368 + mm_tracker: BaseMultiModalItemTracker,
369 +) -> List[ConversationMessage]:
370 role = message["role"]
371 content = message.get("content")
372
@@ -292,7 +386,7 @@ def parse_chat_messages(
386 messages: List[ChatCompletionMessageParam],
387 model_config: ModelConfig,
388 tokenizer: AnyTokenizer,
295 -) -> Tuple[List[ConversationMessage], Optional[Awaitable[MultiModalDataDict]]]:
389 +) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:
390 conversation: List[ConversationMessage] = []
391 mm_tracker = MultiModalItemTracker(model_config, tokenizer)
392
@@ -304,6 +398,22 @@ def parse_chat_messages(
398 return conversation, mm_tracker.all_mm_data()
399
400
401 +def parse_chat_messages_futures(
402 + messages: List[ChatCompletionMessageParam],
403 + model_config: ModelConfig,
404 + tokenizer: AnyTokenizer,
405 +) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
406 + conversation: List[ConversationMessage] = []
407 + mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
408 +
409 + for msg in messages:
410 + sub_messages = _parse_chat_message_content(msg, mm_tracker)
411 +
412 + conversation.extend(sub_messages)
413 +
414 + return conversation, mm_tracker.all_mm_data()
415 +
416 +
417 def apply_chat_template(
418 tokenizer: AnyTokenizer,
419 conversation: List[ConversationMessage],
vllm/entrypoints/llm.py
+19 -12
@@ -23,7 +23,7 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer,
23 get_cached_tokenizer)
24 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
25 from vllm.usage.usage_lib import UsageContext
26 -from vllm.utils import Counter, deprecate_kwargs
26 +from vllm.utils import Counter, deprecate_kwargs, is_list_of
27
28 logger = init_logger(__name__)
29
@@ -358,15 +358,18 @@ class LLM:
358 add_generation_prompt: bool = True,
359 ) -> List[RequestOutput]:
360 """
361 - Generates responses for chat messages.
361 + Generate responses for a chat conversation.
362
363 - Converts the messages to prompts using the tokenizer and calls
364 - the :meth:`generate` method to generate the responses.
363 + The chat conversation is converted into a text prompt using the
364 + tokenizer and calls the :meth:`generate` method to generate the
365 + responses.
366 +
367 + Multi-modal inputs can be passed in the same way you would pass them
368 + to the OpenAI API.
369
370 Args:
367 - messages: A list of messages to generate responses for. Each
368 - message is a list of dictionaries with 'role' and 'content'
369 - keys.
371 + messages: A single conversation represented as a list of messages.
372 + Each message is a dictionary with 'role' and 'content' keys.
373 sampling_params: The sampling parameters for text generation.
374 If None, we use the default sampling parameters. When it
375 is a single value, it is applied to every prompt. When it
@@ -387,21 +390,25 @@ class LLM:
390 tokenizer = self.get_tokenizer()
391 model_config = self.llm_engine.get_model_config()
392
390 - conversations, _ = parse_chat_messages(messages, model_config,
391 - tokenizer)
393 + conversation, mm_data = parse_chat_messages(messages, model_config,
394 + tokenizer)
395
396 prompt = apply_chat_template(
397 tokenizer,
395 - conversations,
398 + conversation,
399 chat_template=chat_template,
397 - add_generation_prompt=add_generation_prompt)
400 + add_generation_prompt=add_generation_prompt,
401 + )
402
403 inputs: PromptInputs
400 - if isinstance(prompt, list) and isinstance(prompt[0], int):
404 + if is_list_of(prompt, int):
405 inputs = TokensPrompt(prompt_token_ids=prompt)
406 else:
407 inputs = TextPrompt(prompt=prompt)
408
409 + if mm_data is not None:
410 + inputs["multi_modal_data"] = mm_data
411 +
412 return self.generate(
413 inputs,
414 sampling_params=sampling_params,
vllm/entrypoints/openai/serving_chat.py
+3 -6
@@ -11,7 +11,7 @@ from vllm.engine.protocol import AsyncEngineClient
11 from vllm.entrypoints.chat_utils import (ConversationMessage,
12 apply_chat_template,
13 load_chat_template,
14 - parse_chat_messages)
14 + parse_chat_messages_futures)
15 from vllm.entrypoints.logger import RequestLogger
16 from vllm.entrypoints.openai.protocol import (
17 ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -26,7 +26,6 @@ from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
26 TextTokensPrompt)
27 from vllm.inputs import TokensPrompt
28 from vllm.logger import init_logger
29 -from vllm.multimodal import MultiModalDataDict
29 from vllm.outputs import RequestOutput
30 from vllm.sequence import Logprob
31 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
@@ -94,7 +93,7 @@ class OpenAIServingChat(OpenAIServing):
93 tokenizer = await self.async_engine_client.get_tokenizer(
94 lora_request)
95
97 - conversation, mm_data_future = parse_chat_messages(
96 + conversation, mm_data_future = parse_chat_messages_futures(
97 request.messages, model_config, tokenizer)
98
99 tool_dicts = None if request.tools is None else [
@@ -114,10 +113,8 @@ class OpenAIServingChat(OpenAIServing):
113 logger.error("Error in applying chat template from request: %s", e)
114 return self.create_error_response(str(e))
115
117 - mm_data: Optional[MultiModalDataDict] = None
116 try:
119 - if mm_data_future:
120 - mm_data = await mm_data_future
117 + mm_data = await mm_data_future
118 except Exception as e:
119 logger.error("Error in loading multi-modal data: %s", e)
120 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_tokenization.py
+4 -3
@@ -4,7 +4,7 @@ from vllm.config import ModelConfig
4 from vllm.engine.protocol import AsyncEngineClient
5 from vllm.entrypoints.chat_utils import (apply_chat_template,
6 load_chat_template,
7 - parse_chat_messages)
7 + parse_chat_messages_futures)
8 from vllm.entrypoints.logger import RequestLogger
9 # yapf conflicts with isort for this block
10 # yapf: disable
@@ -65,10 +65,11 @@ class OpenAIServingTokenization(OpenAIServing):
65 if isinstance(request, TokenizeChatRequest):
66 model_config = self.model_config
67
68 - conversation, mm_data_future = parse_chat_messages(
68 + conversation, mm_data_future = parse_chat_messages_futures(
69 request.messages, model_config, tokenizer)
70
71 - if mm_data_future:
71 + mm_data = await mm_data_future
72 + if mm_data:
73 logger.warning(
74 "Multi-modal inputs are ignored during tokenization")
75
vllm/multimodal/utils.py
+10
@@ -120,6 +120,16 @@ async def async_fetch_audio(
120 return librosa.load(BytesIO(audio_bytes), sr=None)
121
122
123 +def get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
124 + audio, sr = fetch_audio(audio_url)
125 + return {"audio": (audio, sr)}
126 +
127 +
128 +def get_and_parse_image(image_url: str) -> MultiModalDataDict:
129 + image = fetch_image(image_url)
130 + return {"image": image}
131 +
132 +
133 async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
134 audio, sr = await async_fetch_audio(audio_url)
135 return {"audio": (audio, sr)}
vllm/transformers_utils/tokenizers/mistral.py
+3 -2
@@ -52,12 +52,13 @@ class MistralTokenizer:
52 assert isinstance(self.tokenizer,
53 (Tekkenizer, SentencePieceTokenizer)), type(
54 self.tokenizer)
55 - self._is_tekken = isinstance(self.tokenizer, Tekkenizer)
55
57 - if self._is_tekken:
56 + if (is_tekken := isinstance(self.tokenizer, Tekkenizer)):
57 # Make sure special tokens will not raise
58 self.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE
59
60 + self._is_tekken = is_tekken
61 +
62 # the following attributes are set to fit VLLM's design
63 self.is_fast = True
64 self.chat_template = True