A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] Extend beyond image modality and support mixed-modality inference with Llava-OneVision (#11685)
Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Roger Wang committed
Jan 6, 2025 at 11:58 UTC
91b361ae898c944f823534121613f9d3dc19d7d1
17 files changed
+636
-282
docs/source/models/supported_models.md
+1
-1
@@ -647,7 +647,7 @@ See [this page](#generative-models) for more information on how to use generativ
647
- `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc.
648
-
649
- ✅︎
650
- -
650
+ - ✅︎
651
* - `MiniCPMV`
652
- MiniCPM-V
653
- T + I<sup>E+</sup>
tests/multimodal/test_utils.py
+208
-1
@@ -2,16 +2,22 @@ import base64
2
import mimetypes
3
import os
4
from tempfile import NamedTemporaryFile, TemporaryDirectory
5
-from typing import Dict, Tuple
5
+from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
6
7
import numpy as np
8
import pytest
9
from PIL import Image, ImageChops
10
from transformers import AutoConfig, AutoTokenizer
11
12
+from vllm.multimodal.inputs import PlaceholderRange
13
from vllm.multimodal.utils import (MediaConnector,
14
+ merge_and_sort_multimodal_metadata,
15
repeat_and_pad_placeholder_tokens)
16
17
+if TYPE_CHECKING:
18
+ from vllm.multimodal.hasher import MultiModalHashDict
19
+ from vllm.multimodal.inputs import MultiModalPlaceholderDict
20
+
21
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
22
TEST_IMAGE_URLS = [
23
"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
@@ -191,3 +197,204 @@ def test_repeat_and_pad_placeholder_tokens(model):
197
assert new_prompt == expected_prompt
198
assert new_token_ids == expected_token_ids
199
assert ranges == expected_ranges
200
+
201
+
202
+# Used for the next two tests related to `merge_and_sort_multimodal_metadata`.
203
+class TestCase(NamedTuple):
204
+ mm_positions: "MultiModalPlaceholderDict"
205
+ mm_hashes: Optional["MultiModalHashDict"]
206
+ expected_modalities: list[str]
207
+ expected_ranges: list[PlaceholderRange]
208
+ expected_hashes: Optional[list[str]]
209
+
210
+
211
+def test_merge_and_sort_multimodal_metadata():
212
+
213
+ test_cases = [
214
+ # Single modality should return result as is but flattened
215
+ TestCase(
216
+ mm_positions={
217
+ "image": [
218
+ PlaceholderRange(offset=0, length=2),
219
+ PlaceholderRange(offset=3, length=2),
220
+ ]
221
+ },
222
+ mm_hashes={"image": ["hash1", "hash2"]},
223
+ expected_modalities=["image"],
224
+ expected_ranges=[
225
+ PlaceholderRange(offset=0, length=2),
226
+ PlaceholderRange(offset=3, length=2),
227
+ ],
228
+ expected_hashes=["hash1", "hash2"],
229
+ ),
230
+
231
+ # Single modality without hashes return None for mm hash.
232
+ TestCase(
233
+ mm_positions={
234
+ "image": [
235
+ PlaceholderRange(offset=0, length=2),
236
+ PlaceholderRange(offset=2, length=2),
237
+ ]
238
+ },
239
+ mm_hashes=None,
240
+ expected_modalities=["image"],
241
+ expected_ranges=[
242
+ PlaceholderRange(offset=0, length=2),
243
+ PlaceholderRange(offset=2, length=2),
244
+ ],
245
+ expected_hashes=None,
246
+ ),
247
+
248
+ # Multiple modalities with hashes should return sorted modalities
249
+ # and flattened ranges and hashes.
250
+ TestCase(
251
+ mm_positions={
252
+ "image": [
253
+ PlaceholderRange(offset=7, length=4),
254
+ PlaceholderRange(offset=11, length=5),
255
+ ],
256
+ "audio": [
257
+ PlaceholderRange(offset=0, length=2),
258
+ PlaceholderRange(offset=2, length=3),
259
+ ]
260
+ },
261
+ mm_hashes={
262
+ "image": ["image_hash1", "image_hash2"],
263
+ "audio": ["audio_hash1", "audio_hash2"],
264
+ },
265
+ expected_modalities=["audio", "image"],
266
+ expected_ranges=[
267
+ PlaceholderRange(offset=0, length=2),
268
+ PlaceholderRange(offset=2, length=3),
269
+ PlaceholderRange(offset=7, length=4),
270
+ PlaceholderRange(offset=11, length=5),
271
+ ],
272
+ expected_hashes=[
273
+ "audio_hash1", "audio_hash2", "image_hash1", "image_hash2"
274
+ ],
275
+ ),
276
+
277
+ # Multiple modalities without hashes should return sorted modalities
278
+ # and flattened ranges and None.
279
+ TestCase(
280
+ mm_positions={
281
+ "image": [
282
+ PlaceholderRange(offset=7, length=4),
283
+ PlaceholderRange(offset=11, length=5),
284
+ ],
285
+ "audio": [
286
+ PlaceholderRange(offset=0, length=2),
287
+ PlaceholderRange(offset=2, length=3),
288
+ ]
289
+ },
290
+ mm_hashes=None,
291
+ expected_modalities=["audio", "image"],
292
+ expected_ranges=[
293
+ PlaceholderRange(offset=0, length=2),
294
+ PlaceholderRange(offset=2, length=3),
295
+ PlaceholderRange(offset=7, length=4),
296
+ PlaceholderRange(offset=11, length=5),
297
+ ],
298
+ expected_hashes=None,
299
+ ),
300
+
301
+ # Three modalities
302
+ TestCase(
303
+ mm_positions={
304
+ "image": [
305
+ PlaceholderRange(offset=15, length=7),
306
+ PlaceholderRange(offset=22, length=8),
307
+ ],
308
+ "audio": [
309
+ PlaceholderRange(offset=0, length=2),
310
+ ],
311
+ "video": [
312
+ PlaceholderRange(offset=3, length=4),
313
+ PlaceholderRange(offset=7, length=5),
314
+ PlaceholderRange(offset=12, length=6),
315
+ ]
316
+ },
317
+ mm_hashes={
318
+ "image": ["image_hash1", "image_hash2"],
319
+ "audio": ["audio_hash1"],
320
+ "video": ["video_hash1", "video_hash2", "video_hash3"]
321
+ },
322
+ expected_modalities=["audio", "video", "image"],
323
+ expected_ranges=[
324
+ PlaceholderRange(offset=0, length=2),
325
+ PlaceholderRange(offset=3, length=4),
326
+ PlaceholderRange(offset=7, length=5),
327
+ PlaceholderRange(offset=12, length=6),
328
+ PlaceholderRange(offset=15, length=7),
329
+ PlaceholderRange(offset=22, length=8),
330
+ ],
331
+ expected_hashes=[
332
+ "audio_hash1", "video_hash1", "video_hash2", "video_hash3",
333
+ "image_hash1", "image_hash2"
334
+ ],
335
+ ),
336
+ ]
337
+
338
+ for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
339
+ expected_hashes) in test_cases:
340
+ modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
341
+ mm_positions, mm_hashes)
342
+
343
+ assert modalities == expected_modalities
344
+ assert ranges == expected_ranges
345
+ assert hashes == expected_hashes
346
+
347
+
348
+def test_merge_and_sort_multimodal_metadata_with_interleaving():
349
+
350
+ test_cases = [
351
+
352
+ # <image> <audio> <image> <audio>
353
+ TestCase(
354
+ mm_positions={
355
+ "image": [
356
+ PlaceholderRange(offset=0, length=4),
357
+ PlaceholderRange(offset=8, length=2),
358
+ ],
359
+ "audio": [
360
+ PlaceholderRange(offset=5, length=2),
361
+ PlaceholderRange(offset=11, length=4),
362
+ ]
363
+ },
364
+ mm_hashes={
365
+ "image": ["image_hash1", "image_hash2"],
366
+ "audio": ["audio_hash1", "audio_hash2"],
367
+ },
368
+ expected_modalities=[],
369
+ expected_ranges=[],
370
+ expected_hashes=None,
371
+ ),
372
+
373
+ # <image> <image> <video> <audio> <image>
374
+ TestCase(
375
+ mm_positions={
376
+ "image": [
377
+ PlaceholderRange(offset=0, length=2),
378
+ PlaceholderRange(offset=2, length=3),
379
+ PlaceholderRange(offset=20, length=4),
380
+ ],
381
+ "audio": [
382
+ PlaceholderRange(offset=5, length=2),
383
+ ],
384
+ "video": [
385
+ PlaceholderRange(offset=8, length=5),
386
+ ]
387
+ },
388
+ mm_hashes=None,
389
+ expected_modalities=[],
390
+ expected_ranges=[],
391
+ expected_hashes=None,
392
+ ),
393
+ ]
394
+
395
+ for case in test_cases:
396
+ with pytest.raises(ValueError) as ex_info:
397
+ merge_and_sort_multimodal_metadata(case.mm_positions,
398
+ case.mm_hashes)
399
+
400
+ assert "Interleaved mixed-modality" in str(ex_info.value)
tests/v1/core/test_kv_cache_utils.py
+11
-7
@@ -1,6 +1,6 @@
1
import pytest
2
3
-from vllm.inputs import token_inputs
3
+from vllm.multimodal.inputs import MultiModalKwargs
4
from vllm.sampling_params import SamplingParams
5
from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
6
KVCacheBlock,
@@ -14,14 +14,18 @@ def make_request(request_id,
14
prompt_token_ids,
15
mm_positions=None,
16
mm_hashes=None):
17
+ if mm_positions is None:
18
+ multi_modal_inputs = None
19
+ else:
20
+ multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
21
+
22
return Request(
23
request_id=request_id,
19
- inputs=token_inputs(
20
- prompt_token_ids=prompt_token_ids,
21
- multi_modal_placeholders={"image": mm_positions}
22
- if mm_positions else None,
23
- multi_modal_hashes=mm_hashes,
24
- ),
24
+ prompt=None,
25
+ prompt_token_ids=prompt_token_ids,
26
+ multi_modal_inputs=multi_modal_inputs,
27
+ multi_modal_hashes=mm_hashes,
28
+ multi_modal_placeholders=mm_positions,
29
sampling_params=SamplingParams(max_tokens=17),
30
eos_token_id=100,
31
arrival_time=0,
tests/v1/core/test_prefix_caching.py
+11
-6
@@ -1,8 +1,7 @@
1
"""Compare the with and without prefix caching."""
2
import pytest
3
4
-from vllm.inputs import token_inputs
5
-from vllm.multimodal.inputs import PlaceholderRange
4
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
5
from vllm.sampling_params import SamplingParams
6
from vllm.utils import cdiv
7
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
@@ -13,12 +12,18 @@ def make_request(request_id,
12
prompt_token_ids,
13
mm_positions=None,
14
mm_hashes=None):
15
+ if mm_positions is None:
16
+ multi_modal_inputs = None
17
+ else:
18
+ multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
19
+
20
return Request(
21
request_id=request_id,
18
- inputs=token_inputs(prompt_token_ids=prompt_token_ids,
19
- multi_modal_placeholders={"image": mm_positions}
20
- if mm_positions else None,
21
- multi_modal_hashes=mm_hashes),
22
+ prompt=None,
23
+ prompt_token_ids=prompt_token_ids,
24
+ multi_modal_inputs=multi_modal_inputs,
25
+ multi_modal_hashes=mm_hashes,
26
+ multi_modal_placeholders=mm_positions,
27
sampling_params=SamplingParams(max_tokens=17),
28
eos_token_id=100,
29
arrival_time=0,
vllm/model_executor/models/interfaces.py
+5
-1
@@ -39,8 +39,12 @@ class SupportsMultiModal(Protocol):
39
40
The output embeddings must be one of the following formats:
41
- A list or tuple of 2D tensors, where each tensor corresponds to
42
- each input image.
42
+ each input multimodal data item (e.g, image).
43
- A single 3D tensor, with the batch dimension grouping the 2D tensors.
44
+
45
+ NOTE: The returned multimodal embeddings must be in the same order as
46
+ the appearances of their corresponding multimodal data item in the
47
+ input prompt.
48
"""
49
...
50
vllm/model_executor/models/llava_onevision.py
+33
-32
@@ -35,6 +35,9 @@ from .siglip import SiglipVisionModel
35
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
maybe_prefix, merge_multimodal_embeddings)
37
38
+# For profile run
39
+_MAX_FRAMES_PER_VIDEO = 16
40
+
41
42
class LlavaOnevisionVideoPixelInputs(TypedDict):
43
type: Literal["pixel_values_videos"]
@@ -223,8 +226,10 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
226
max_image_tokens = self._get_max_image_tokens() * max_images
227
max_total_frames = self._get_max_video_frames(seq_len -
228
max_image_tokens)
229
+ max_frames_per_video = min(max_total_frames // max(max_videos, 1),
230
+ _MAX_FRAMES_PER_VIDEO)
231
227
- return max(max_total_frames // max(max_videos, 1), 1)
232
+ return max(max_frames_per_video, 1)
233
234
def _get_max_video_tokens(self, seq_len: int) -> int:
235
target_width, target_height = self._get_image_size_with_most_features()
@@ -558,13 +563,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
563
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
564
modalities = {}
565
561
- if "pixel_values" in kwargs:
562
- modalities["images"] = self._parse_and_validate_image_input(
563
- **kwargs)
564
-
565
- if "pixel_values_videos" in kwargs:
566
- modalities["videos"] = self._parse_and_validate_video_input(
567
- **kwargs)
566
+ # Preserve the order of modalities if there are multiple of them
567
+ # from the order of kwargs.
568
+ for input_key in kwargs:
569
+ if input_key == "pixel_values" and "images" not in modalities:
570
+ modalities["images"] = self._parse_and_validate_image_input(
571
+ **kwargs)
572
+ if input_key == "pixel_values_videos" and "videos" not in modalities: # noqa E501
573
+ modalities["videos"] = self._parse_and_validate_video_input(
574
+ **kwargs)
575
576
return modalities
577
@@ -824,21 +831,21 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
831
if not modalities:
832
return None
833
827
- # We make a tuple of each embedding with its modality string. This is a
828
- # temporary workaround for models to handle mixed modalities when
829
- # get_multimodal_embeddings and get_input_embeddings are called
830
- # separately.
831
- # TODO(ywang96): Add support for mixed-modality inference for v1.
832
- multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
833
-
834
- if "images" in modalities:
835
- image_input = modalities["images"]
836
- vision_embeddings = self._process_image_input(image_input)
837
- multimodal_embeddings.append((vision_embeddings, "image"))
838
- if "videos" in modalities:
839
- video_input = modalities["videos"]
840
- video_embeddings = self._process_video_pixels(video_input)
841
- multimodal_embeddings.append((video_embeddings, "video"))
834
+ # The result multimodal_embeddings is tuple of tensors, with each
835
+ # tensor correspoending to a multimodal data item (image or video).
836
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
837
+
838
+ # NOTE: It is important to iterate over the keys in this dictionary
839
+ # to preserve the order of the modalities.
840
+ for modality in modalities:
841
+ if modality == "images":
842
+ image_input = modalities["images"]
843
+ vision_embeddings = self._process_image_input(image_input)
844
+ multimodal_embeddings += tuple(vision_embeddings)
845
+ if modality == "videos":
846
+ video_input = modalities["videos"]
847
+ video_embeddings = self._process_video_pixels(video_input)
848
+ multimodal_embeddings += tuple(video_embeddings)
849
850
return multimodal_embeddings
851
@@ -850,15 +857,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
857
) -> torch.Tensor:
858
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
859
if multimodal_embeddings is not None:
853
- for embeddings, modality in multimodal_embeddings:
854
- if modality == "image":
855
- inputs_embeds = merge_multimodal_embeddings(
856
- input_ids, inputs_embeds, embeddings,
857
- self.config.image_token_index)
858
- if modality == "video":
859
- inputs_embeds = merge_multimodal_embeddings(
860
- input_ids, inputs_embeds, embeddings,
861
- self.config.video_token_index)
860
+ inputs_embeds = merge_multimodal_embeddings(
861
+ input_ids, inputs_embeds, multimodal_embeddings,
862
+ [self.config.image_token_index, self.config.video_token_index])
863
return inputs_embeds
864
865
def forward(
vllm/model_executor/models/molmo.py
-3
@@ -972,8 +972,6 @@ def image_input_mapper_for_molmo(
972
assert len(data) == 1, "Molmo supports only one image per prompt."
973
data = data[0]
974
975
- # Remove unused dummy PIL image
976
- data.pop('raw_mm_data', None)
975
return MultiModalKwargs(data)
976
977
@@ -1019,7 +1017,6 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
1017
dummy_imgdata = {
1018
"images": out["images"],
1019
"image_input_idx": out["image_input_idx"],
1022
- "raw_mm_data": dummy_image,
1020
}
1021
if "image_masks" in out:
1022
dummy_imgdata["image_masks"] = out["image_masks"]
vllm/multimodal/__init__.py
+3
@@ -1,4 +1,5 @@
1
from .base import MultiModalPlaceholderMap, MultiModalPlugin
2
+from .hasher import MultiModalHashDict, MultiModalHasher
3
from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
4
MultiModalDataDict, MultiModalKwargs,
5
MultiModalPlaceholderDict, NestedTensors)
@@ -18,6 +19,8 @@ __all__ = [
19
"ModalityData",
20
"MultiModalDataBuiltins",
21
"MultiModalDataDict",
22
+ "MultiModalHashDict",
23
+ "MultiModalHasher",
24
"MultiModalKwargs",
25
"MultiModalPlaceholderDict",
26
"MultiModalPlaceholderMap",
vllm/multimodal/hasher.py
new
+100
@@ -0,0 +1,100 @@
1
+import pickle
2
+from typing import TYPE_CHECKING, Iterable, Mapping, Optional
3
+
4
+import numpy as np
5
+import torch
6
+from blake3 import blake3
7
+from PIL import Image
8
+
9
+from vllm.logger import init_logger
10
+
11
+if TYPE_CHECKING:
12
+ from vllm.inputs import TokensPrompt
13
+
14
+logger = init_logger(__name__)
15
+
16
+MultiModalHashDict = Mapping[str, list[str]]
17
+"""
18
+A dictionary containing hashes for items in each modality.
19
+"""
20
+
21
+
22
+class MultiModalHasher:
23
+
24
+ @classmethod
25
+ def serialize_item(cls, obj: object) -> bytes:
26
+ # Simple cases
27
+ if isinstance(obj, str):
28
+ return obj.encode("utf-8")
29
+ if isinstance(obj, bytes):
30
+ return obj
31
+ if isinstance(obj, Image.Image):
32
+ return obj.tobytes()
33
+
34
+ # Convertible to NumPy arrays
35
+ if isinstance(obj, torch.Tensor):
36
+ obj = obj.numpy()
37
+ if isinstance(obj, (int, float)):
38
+ obj = np.array(obj)
39
+ if isinstance(obj, np.ndarray):
40
+ return obj.tobytes()
41
+
42
+ logger.warning(
43
+ "No serialization method found for %s. "
44
+ "Falling back to pickle.", type(obj))
45
+
46
+ return pickle.dumps(obj)
47
+
48
+ @classmethod
49
+ def item_to_bytes(
50
+ cls,
51
+ key: str,
52
+ obj: object,
53
+ ) -> Iterable[tuple[bytes, bytes]]:
54
+ # Recursive cases
55
+ if isinstance(obj, (list, tuple)):
56
+ for i, elem in enumerate(obj):
57
+ yield from cls.item_to_bytes(f"{key}.{i}", elem)
58
+ elif isinstance(obj, dict):
59
+ for k, v in obj.items():
60
+ yield from cls.item_to_bytes(f"{key}.{k}", v)
61
+ else:
62
+ key_bytes = cls.serialize_item(key)
63
+ value_bytes = cls.serialize_item(obj)
64
+ yield key_bytes, value_bytes
65
+
66
+ @classmethod
67
+ def hash_kwargs(cls, **kwargs: object) -> str:
68
+ hasher = blake3()
69
+
70
+ for k, v in kwargs.items():
71
+ for k_bytes, v_bytes in cls.item_to_bytes(k, v):
72
+ hasher.update(k_bytes)
73
+ hasher.update(v_bytes)
74
+
75
+ return hasher.hexdigest()
76
+
77
+ @classmethod
78
+ def hash_prompt_mm_data(
79
+ cls, prompt: "TokensPrompt") -> Optional["MultiModalHashDict"]:
80
+ """Hash multimodal data in the user input prompt if they exist."""
81
+
82
+ if "multi_modal_data" not in prompt:
83
+ return None
84
+
85
+ mm_data = prompt["multi_modal_data"]
86
+ if not mm_data:
87
+ # mm_data can be None or an empty dict.
88
+ return None
89
+
90
+ mm_items = {
91
+ modality: items if isinstance(items, list) else [items]
92
+ for modality, items in mm_data.items()
93
+ }
94
+
95
+ mm_hashes = {
96
+ modality: [cls.hash_kwargs(**{modality: item}) for item in items]
97
+ for modality, items in mm_items.items()
98
+ }
99
+
100
+ return mm_hashes
vllm/multimodal/inputs.py
+6
-3
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2
from collections import UserDict, defaultdict
3
from collections.abc import Mapping, Sequence
4
from dataclasses import dataclass
5
-from typing import (Any, Literal, Optional, TypedDict, TypeVar, Union, cast,
6
- final)
5
+from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
6
+ Union, cast, final)
7
8
import numpy as np
9
import torch
@@ -14,6 +14,9 @@ from typing_extensions import NotRequired, TypeAlias
14
15
from vllm.utils import JSONTree, full_groupby, is_list_of, json_map_leaves
16
17
+if TYPE_CHECKING:
18
+ from .hasher import MultiModalHashDict
19
+
20
_T = TypeVar("_T")
21
22
HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
@@ -513,7 +516,7 @@ class MultiModalInputsV2(TypedDict):
516
mm_kwargs: MultiModalKwargs
517
"""Keyword arguments to be directly passed to the model after batching."""
518
516
- mm_hashes: NotRequired[list[str]]
519
+ mm_hashes: NotRequired[Optional["MultiModalHashDict"]]
520
"""The hashes of the multi-modal data."""
521
522
mm_placeholders: MultiModalPlaceholderDict
vllm/multimodal/processing.py
+30
-62
@@ -1,4 +1,3 @@
1
-import pickle
1
import re
2
from abc import ABC, abstractmethod
3
from collections import defaultdict
@@ -7,18 +6,16 @@ from dataclasses import dataclass, field
6
from functools import lru_cache
7
from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
10
-import numpy as np
11
-import torch
12
-from blake3 import blake3
13
-from PIL import Image
9
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
10
11
+from vllm import envs
12
from vllm.inputs import DummyData, InputProcessingContext
13
from vllm.logger import init_logger
14
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
15
encode_tokens)
16
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
17
18
+from .hasher import MultiModalHasher
19
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
20
MultiModalInputsV2, MultiModalKwargs,
21
MultiModalKwargsItem, PlaceholderRange)
@@ -486,56 +483,6 @@ class ProcessingCache:
483
logger.debug("ProcessingCache: hit_ratio = %.2f",
484
cache_stats.hit_ratio)
485
489
- def _serialize_item(self, obj: object) -> bytes:
490
- # Simple cases
491
- if isinstance(obj, str):
492
- return obj.encode("utf-8")
493
- if isinstance(obj, bytes):
494
- return obj
495
- if isinstance(obj, Image.Image):
496
- return obj.tobytes()
497
-
498
- # Convertible to NumPy arrays
499
- if isinstance(obj, torch.Tensor):
500
- obj = obj.numpy()
501
- if isinstance(obj, (int, float)):
502
- obj = np.array(obj)
503
- if isinstance(obj, np.ndarray):
504
- return obj.tobytes()
505
-
506
- logger.warning(
507
- "No serialization method found for %s. "
508
- "Falling back to pickle.", type(obj))
509
-
510
- return pickle.dumps(obj)
511
-
512
- def _item_to_bytes(
513
- self,
514
- key: str,
515
- obj: object,
516
- ) -> Iterable[tuple[bytes, bytes]]:
517
- # Recursive cases
518
- if isinstance(obj, (list, tuple)):
519
- for i, elem in enumerate(obj):
520
- yield from self._item_to_bytes(f"{key}.{i}", elem)
521
- elif isinstance(obj, dict):
522
- for k, v in obj.items():
523
- yield from self._item_to_bytes(f"{key}.{k}", v)
524
- else:
525
- key_bytes = self._serialize_item(key)
526
- value_bytes = self._serialize_item(obj)
527
- yield key_bytes, value_bytes
528
-
529
- def _hash_kwargs(self, **kwargs: object) -> str:
530
- hasher = blake3()
531
-
532
- for k, v in kwargs.items():
533
- for k_bytes, v_bytes in self._item_to_bytes(k, v):
534
- hasher.update(k_bytes)
535
- hasher.update(v_bytes)
536
-
537
- return hasher.hexdigest()
538
-
486
def get(
487
self,
488
model_id: str,
@@ -554,9 +501,9 @@ class ProcessingCache:
501
"""
502
self._maybe_log_cache_stats()
503
557
- cache_key = self._hash_kwargs(model_id=model_id,
558
- **{modality: input_item},
559
- **input_kwargs)
504
+ cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
505
+ **{modality: input_item},
506
+ **input_kwargs)
507
return self._cache.get(cache_key)
508
509
def put(
@@ -571,9 +518,9 @@ class ProcessingCache:
518
Put a processed multi-modal item into the cache
519
according to its dependencies (see :meth:`get`).
520
"""
574
- cache_key = self._hash_kwargs(model_id=model_id,
575
- **{modality: input_item},
576
- **input_kwargs)
521
+ cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
522
+ **{modality: input_item},
523
+ **input_kwargs)
524
self._cache.put(cache_key, output_kwargs)
525
526
@@ -1049,6 +996,24 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
996
"""
997
mm_items = self._to_mm_items(mm_data)
998
999
+ # Create MM hashes (only used in V1)
1000
+ # TODO: Use these hash keys for caching operations in apply_hf_processor
1001
+ # instead of rehashing.
1002
+
1003
+ if envs.VLLM_USE_V1:
1004
+ model_id = self.ctx.model_config.model
1005
+ mm_hashes = {
1006
+ modality: [
1007
+ MultiModalHasher.hash_kwargs(model_id=model_id,
1008
+ **{modality: item},
1009
+ **hf_processor_mm_kwargs)
1010
+ for item in items
1011
+ ]
1012
+ for modality, items in mm_items.items()
1013
+ }
1014
+ else:
1015
+ mm_hashes = None
1016
+
1017
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1018
prompt_text,
1019
mm_items,
@@ -1122,6 +1087,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1087
prompt=prompt_text,
1088
prompt_token_ids=prompt_ids,
1089
mm_kwargs=mm_kwargs,
1090
+ mm_hashes=mm_hashes,
1091
mm_placeholders=mm_placeholder_ranges,
1092
)
1093
@@ -1174,7 +1140,9 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1140
"tokens.")
1141
1142
total_len = len(prompt_token_ids)
1177
- if total_len > seq_len:
1143
+
1144
+ # V0 does not support chunked prefill.
1145
+ if total_len > seq_len and not envs.VLLM_USE_V1:
1146
logger.warning(
1147
"The context length (%d) of the model is too short "
1148
"to hold the multi-modal embeddings in the worst case "
vllm/multimodal/utils.py
+85
-1
@@ -1,6 +1,6 @@
1
from functools import lru_cache
2
from pathlib import Path
3
-from typing import Optional, TypeVar, Union
3
+from typing import TYPE_CHECKING, Optional, TypeVar, Union
4
from urllib.parse import ParseResult, urlparse
5
6
import numpy as np
@@ -25,6 +25,10 @@ cached_get_tokenizer = lru_cache(get_tokenizer)
25
26
_M = TypeVar("_M")
27
28
+if TYPE_CHECKING:
29
+ from .hasher import MultiModalHashDict
30
+ from .inputs import MultiModalPlaceholderDict
31
+
32
33
class MediaConnector:
34
@@ -437,3 +441,83 @@ def consecutive_placeholder_ranges(
441
PlaceholderRange(offset=initial_offset + i * item_size,
442
length=item_size) for i in range(num_items)
443
]
444
+
445
+
446
+def merge_and_sort_multimodal_metadata(
447
+ mm_positions: "MultiModalPlaceholderDict",
448
+ mm_hashes: Optional["MultiModalHashDict"],
449
+) -> tuple[list[str], list[PlaceholderRange], Optional[list[str]]]:
450
+ """Given a MultiModalPlaceholderDict, merge all PlaceholderRange
451
+ objects from all available modalities into a single list of
452
+ PlaceholderRange, sorted by their offset (starting index in the input
453
+ sequence) in the ascending order.
454
+
455
+ Optionally if a MultiModalHashDict is given, same operation will be
456
+ applied to the object and the sorted list of hashes will be returned.
457
+
458
+ Raises:
459
+ ValueError: If the input prompt has interleaved placeholders from
460
+ different modalities (e.g, "<image><audio><image> Describe the
461
+ content.")
462
+
463
+ Returns:
464
+ list[str]: Sorted list of involved modalities.
465
+ list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
466
+ mm_positions.
467
+ Optional[list[str]]: Sorted list of all hashes from mm_hashes if
468
+ given, None otherwise.
469
+ """
470
+
471
+ modalities = list(mm_positions.keys())
472
+
473
+ assert len(modalities) > 0, "No modalities found in the mm_positions."
474
+
475
+ # For single modality, placeholder ranges and hashes are already sorted
476
+ # so we can return the list directly.
477
+ if len(modalities) == 1:
478
+ if mm_hashes is None:
479
+ return modalities, list(mm_positions[modalities[0]]), None
480
+ else:
481
+ return modalities, list(mm_positions[modalities[0]]), list(
482
+ mm_hashes[modalities[0]])
483
+
484
+ placeholder_lists_with_modality = [(modality, mm_positions[modality])
485
+ for modality in modalities]
486
+
487
+ if mm_hashes is None:
488
+ sorted_placeholder_lists = sorted(placeholder_lists_with_modality,
489
+ key=lambda x: x[1][0]['offset'])
490
+ sorted_hash_lists = None
491
+ else:
492
+ hashes_lists = [
493
+ mm_hashes[modality] for modality in modalities
494
+ if modality in mm_hashes
495
+ ]
496
+ sorted_pairs = sorted(zip(placeholder_lists_with_modality,
497
+ hashes_lists),
498
+ key=lambda x: x[0][1][0]['offset'])
499
+ sorted_placeholder_tuple, sorted_hash_tuple = zip(*sorted_pairs)
500
+ sorted_placeholder_lists = list(sorted_placeholder_tuple)
501
+ sorted_hash_lists = list(sorted_hash_tuple)
502
+
503
+ sorted_modalities = [modality for modality, _ in sorted_placeholder_lists]
504
+
505
+ # Flatten sorted list of lists to a single list and verify there is no
506
+ # interleaving of placeholders from different modalities.
507
+ merged_placeholders: list[PlaceholderRange] = []
508
+ for modality, placeholder_list in sorted_placeholder_lists:
509
+ if merged_placeholders and placeholder_list[0][
510
+ 'offset'] < merged_placeholders[-1]['offset']:
511
+ raise ValueError(
512
+ "Interleaved mixed-modality inference is currently not "
513
+ "supported.")
514
+ merged_placeholders.extend(placeholder_list)
515
+
516
+ if sorted_hash_lists is not None:
517
+ merged_hashes = []
518
+ for hash_list in sorted_hash_lists:
519
+ merged_hashes.extend(hash_list)
520
+ else:
521
+ merged_hashes = None
522
+
523
+ return sorted_modalities, merged_placeholders, merged_hashes
vllm/v1/engine/__init__.py
+10
-8
@@ -1,12 +1,14 @@
1
import enum
2
from dataclasses import dataclass
3
-from typing import List, Optional, Union
3
+from typing import TYPE_CHECKING, List, Optional, Union
4
5
import msgspec
6
7
-from vllm.lora.request import LoRARequest
8
-from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9
-from vllm.sampling_params import SamplingParams
7
+if TYPE_CHECKING:
8
+ from vllm.lora.request import LoRARequest
9
+ from vllm.multimodal import MultiModalKwargs
10
+ from vllm.multimodal.inputs import PlaceholderRange
11
+ from vllm.sampling_params import SamplingParams
12
13
14
@dataclass
@@ -21,13 +23,13 @@ class EngineCoreRequest:
23
# always be tokenized?
24
prompt: Optional[str]
25
prompt_token_ids: List[int]
24
- mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
26
+ mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
27
mm_hashes: Optional[List[str]]
26
- mm_placeholders: Optional[MultiModalPlaceholderDict]
27
- sampling_params: SamplingParams
28
+ mm_placeholders: Optional[List["PlaceholderRange"]]
29
+ sampling_params: "SamplingParams"
30
eos_token_id: Optional[int]
31
arrival_time: float
30
- lora_request: Optional[LoRARequest]
32
+ lora_request: Optional["LoRARequest"]
33
34
35
class EngineCoreOutput(
vllm/v1/engine/mm_input_mapper.py
-67
@@ -1,10 +1,6 @@
1
from typing import Any, Dict, List, Optional
2
3
-import PIL
4
-from blake3 import blake3
5
-
3
from vllm.config import ModelConfig
7
-from vllm.inputs import PromptType
4
from vllm.logger import init_logger
5
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
6
MultiModalKwargs, MultiModalRegistry)
@@ -144,66 +140,3 @@ class MMInputMapperServer:
140
full_mm_inputs.append(mm_input)
141
142
return full_mm_inputs
147
-
148
-
149
-class MMHasher:
150
-
151
- def __init__(self):
152
- pass
153
-
154
- def hash_dummy_mm_data(
155
- self,
156
- mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157
- """Hash user-defined dummy multimodal data used for profiling."""
158
-
159
- if mm_data is None:
160
- return None
161
-
162
- image_inputs = mm_data['image']
163
-
164
- # This is a temporary workaround for models (e.g, Molmo) that
165
- # process multimodal data in the input processor (therefore
166
- # image_inputs is MultiModalKwargs instead of raw input format).
167
- # `raw_mm_data` with the original input format is expected
168
- # in this case.
169
- if isinstance(image_inputs, dict):
170
- assert "raw_mm_data" in image_inputs and isinstance(
171
- image_inputs["raw_mm_data"], PIL.Image.Image)
172
- image_inputs = image_inputs.pop("raw_mm_data")
173
-
174
- return self.hash_images(image_inputs)
175
-
176
- def hash_prompt_mm_data(self, prompt: PromptType) -> Optional[List[str]]:
177
- """Hash multimodal data in the user input prompt if they exist."""
178
-
179
- if "multi_modal_data" not in prompt:
180
- return None
181
-
182
- mm_data = prompt["multi_modal_data"]
183
- if not mm_data:
184
- # mm_data can be None or an empty dict.
185
- return None
186
-
187
- image_inputs = mm_data["image"]
188
-
189
- return self.hash_images(image_inputs)
190
-
191
- def hash_images(self, image_inputs) -> Optional[List[str]]:
192
- """Hash PIL image objects to strings."""
193
- if not isinstance(image_inputs, list):
194
- image_inputs = [image_inputs]
195
- assert len(image_inputs) > 0
196
-
197
- ret = []
198
- for image in image_inputs:
199
- assert isinstance(image, PIL.Image.Image)
200
-
201
- # Convert image to bytes
202
- bytes = image.tobytes()
203
-
204
- # Hash image bytes
205
- hasher = blake3()
206
- hasher.update(bytes)
207
- ret.append(hasher.hexdigest())
208
-
209
- return ret
vllm/v1/engine/processor.py
+74
-27
@@ -7,14 +7,15 @@ from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
7
from vllm.inputs.parse import is_encoder_decoder_inputs
8
from vllm.inputs.preprocess import InputPreprocessor
9
from vllm.lora.request import LoRARequest
10
-from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
11
- MultiModalRegistry)
10
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalHasher,
11
+ MultiModalKwargs, MultiModalRegistry)
12
+from vllm.multimodal.utils import merge_and_sort_multimodal_metadata
13
from vllm.pooling_params import PoolingParams
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sampling_params import SamplingParams
16
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
17
from vllm.v1.engine import EngineCoreRequest
17
-from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
18
+from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
19
20
21
class Processor:
@@ -47,7 +48,6 @@ class Processor:
48
# Multi-modal hasher (for images)
49
self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
50
cache_config.enable_prefix_caching
50
- self.mm_hasher = MMHasher()
51
52
def process_inputs(
53
self,
@@ -73,11 +73,6 @@ class Processor:
73
assert priority == 0, "vLLM V1 does not support priority at the moment."
74
assert trace_headers is None, "vLLM V1 does not support tracing yet."
75
76
- # Compute MM hashes (if enabled)
77
- mm_hashes = None
78
- if self.use_hash:
79
- mm_hashes = self.mm_hasher.hash_prompt_mm_data(prompt)
80
-
76
# Process inputs.
77
preprocessed_inputs = self.input_preprocessor.preprocess(
78
prompt,
@@ -108,8 +103,20 @@ class Processor:
103
sampling_params.update_from_generation_config(
104
self.generation_config_fields, eos_token_id)
105
106
+ # Multimodal related.
107
+ # Compute MM hashes (if enabled)
108
+ mm_hashes = None
109
+ if self.use_hash:
110
+ # Use mm_hashes from processed inputs if the model has merged
111
+ # input processor.
112
+ if decoder_inputs.multi_modal_hashes:
113
+ mm_hashes = decoder_inputs.multi_modal_hashes
114
+ # Fallback to using MultiModalHasher directly.
115
+ else:
116
+ mm_hashes = MultiModalHasher.hash_prompt_mm_data(prompt)
117
+
118
# For merged preprocessor, mm_data is already mm_inputs
112
- precomputed_mm_inputs = None
119
+ precomputed_mm_inputs: Optional[list[MultiModalKwargs]] = None
120
decoder_mm_data = decoder_inputs.multi_modal_data
121
if isinstance(decoder_mm_data, MultiModalKwargs):
122
# The output of merged multi-modal processor (`decoder_mm_data`)
@@ -122,27 +129,67 @@ class Processor:
129
for item in decoder_mm_data.get_items(modality)
130
]
131
125
- # Apply MM mapper
126
- mm_inputs = None
127
- if len(decoder_mm_data) > 0:
128
- mm_inputs = self.mm_input_mapper_client.process_inputs(
129
- decoder_mm_data,
132
+ mm_positions = decoder_inputs.multi_modal_placeholders
133
+
134
+ # Last-mile processing of multimodal metadata and inputs.
135
+ if mm_positions:
136
+
137
+ # Merge and flatten multimodal placeholders, hashes and inputs
138
+ # from dictionaries to lists, and sort them by each item's position
139
+ # in the input sequence.
140
+ # NOTE: interleaved modalities are not supported.
141
+ (
142
+ sorted_modalities,
143
+ sorted_mm_positions,
144
+ sorted_mm_hashes,
145
+ ) = merge_and_sort_multimodal_metadata(
146
+ mm_positions,
147
mm_hashes,
131
- decoder_inputs.mm_processor_kwargs,
132
- precomputed_mm_inputs,
148
)
149
150
+ # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple
151
+ # modalities involved AND the model supports merged input processor.
152
+ if len(sorted_modalities) > 1 and precomputed_mm_inputs:
153
+
154
+ modality_order_dict = {
155
+ modality: order
156
+ for order, modality in enumerate(sorted_modalities)
157
+ }
158
+
159
+ # Sanity check to make sure each multimodal input has only one
160
+ # modality key.
161
+ for mm_input in precomputed_mm_inputs:
162
+ assert len(mm_input.modalities) == 1
163
+
164
+ # Sort MultiModalKwags to match sorted_mm_positions
165
+ precomputed_mm_inputs = sorted(
166
+ precomputed_mm_inputs,
167
+ key=lambda mm_input: modality_order_dict[list(
168
+ mm_input.modalities)[0]])
169
+
170
+ # Apply mm input cache update (and input mapper if necessary).
171
+ sorted_mm_inputs = self.mm_input_mapper_client.process_inputs(
172
+ mm_data=decoder_mm_data,
173
+ mm_hashes=sorted_mm_hashes,
174
+ mm_processor_kwargs=decoder_inputs.mm_processor_kwargs,
175
+ precomputed_mm_inputs=precomputed_mm_inputs,
176
+ )
177
+ else:
178
+ sorted_mm_inputs = None
179
+ sorted_mm_hashes = None
180
+ sorted_mm_positions = None
181
+
182
return EngineCoreRequest(
136
- request_id,
137
- decoder_inputs.prompt,
138
- decoder_inputs.prompt_token_ids,
139
- mm_inputs,
140
- mm_hashes,
141
- decoder_inputs.multi_modal_placeholders,
142
- sampling_params,
143
- eos_token_id,
144
- arrival_time,
145
- lora_request,
183
+ request_id=request_id,
184
+ prompt=decoder_inputs.prompt,
185
+ prompt_token_ids=decoder_inputs.prompt_token_ids,
186
+ mm_inputs=sorted_mm_inputs,
187
+ mm_hashes=sorted_mm_hashes,
188
+ mm_placeholders=sorted_mm_positions,
189
+ sampling_params=sampling_params,
190
+ eos_token_id=eos_token_id,
191
+ arrival_time=arrival_time,
192
+ lora_request=lora_request,
193
)
194
195
def _validate_model_inputs(self, inputs: ProcessorInputs):
vllm/v1/request.py
+21
-27
@@ -1,15 +1,15 @@
1
import enum
2
from typing import TYPE_CHECKING, List, Optional, Union
3
4
-from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
4
from vllm.lora.request import LoRARequest
6
-from vllm.multimodal import MultiModalKwargs
5
from vllm.sampling_params import SamplingParams
6
from vllm.sequence import RequestMetrics
7
from vllm.v1.engine import EngineCoreRequest
8
from vllm.v1.utils import ConstantList
9
10
if TYPE_CHECKING:
11
+ from vllm.multimodal import MultiModalKwargs
12
+ from vllm.multimodal.inputs import PlaceholderRange
13
from vllm.v1.core.kv_cache_utils import BlockHashType
14
15
@@ -18,14 +18,17 @@ class Request:
18
def __init__(
19
self,
20
request_id: str,
21
- inputs: DecoderOnlyInputs,
21
+ prompt: Optional[str],
22
+ prompt_token_ids: List[int],
23
+ multi_modal_inputs: Optional[List["MultiModalKwargs"]],
24
+ multi_modal_hashes: Optional[List[str]],
25
+ multi_modal_placeholders: Optional[List["PlaceholderRange"]],
26
sampling_params: SamplingParams,
27
eos_token_id: Optional[int],
28
arrival_time: float,
29
lora_request: Optional[LoRARequest] = None,
30
) -> None:
31
self.request_id = request_id
28
- self.inputs = SingletonInputsAdapter(inputs)
32
self.sampling_params = sampling_params
33
# Because of LoRA, the eos token id can be different for each request.
34
self.eos_token_id = eos_token_id
@@ -41,26 +44,21 @@ class Request:
44
assert sampling_params.max_tokens is not None
45
self.max_tokens = sampling_params.max_tokens
46
44
- self.prompt = self.inputs.prompt
45
- self.prompt_token_ids = self.inputs.prompt_token_ids
47
+ self.prompt = prompt
48
+ self.prompt_token_ids = prompt_token_ids
49
self.num_prompt_tokens = len(self.prompt_token_ids)
50
self._output_token_ids: List[int] = []
51
self._all_token_ids: List[int] = self.prompt_token_ids.copy()
52
self.num_computed_tokens = 0
53
51
- # Multi-modal input metadata.
52
- mm_positions = self.inputs.multi_modal_placeholders
53
- if mm_positions:
54
- # FIXME(woosuk): Support other modalities.
55
- self.mm_positions = mm_positions.get("image", [])
56
- else:
57
- self.mm_positions = []
58
- # Output of the mm input mapper (e.g., image tensors).
59
- self.mm_inputs: List[MultiModalKwargs] = []
60
- if self.inputs.multi_modal_inputs:
61
- self.mm_inputs = self.inputs.multi_modal_inputs
54
+ # Multi-modal related
55
+ self.mm_positions = multi_modal_placeholders or []
56
+ self.mm_inputs = multi_modal_inputs or []
57
+ self.mm_hashes: List[str] = multi_modal_hashes or []
58
63
- self.mm_hashes: List[str] = self.inputs.multi_modal_hashes
59
+ # Sanity check
60
+ assert len(self.mm_inputs) == len(self.mm_positions)
61
+ assert len(self.mm_inputs) == len(self.mm_hashes)
62
63
# Cache the computed kv block hashes of the request to avoid
64
# recomputing.
@@ -70,15 +68,11 @@ class Request:
68
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
69
return cls(
70
request_id=request.request_id,
73
- inputs=token_inputs(
74
- prompt_token_ids=request.prompt_token_ids,
75
- prompt=request.prompt,
76
- multi_modal_data=None,
77
- multi_modal_inputs=request.mm_inputs,
78
- multi_modal_hashes=request.mm_hashes,
79
- multi_modal_placeholders=request.mm_placeholders,
80
- mm_processor_kwargs=None,
81
- ),
71
+ prompt=request.prompt,
72
+ prompt_token_ids=request.prompt_token_ids,
73
+ multi_modal_inputs=request.mm_inputs,
74
+ multi_modal_hashes=request.mm_hashes,
75
+ multi_modal_placeholders=request.mm_placeholders,
76
sampling_params=request.sampling_params,
77
eos_token_id=request.eos_token_id,
78
arrival_time=request.arrival_time,
vllm/v1/worker/gpu_model_runner.py
+38
-36
@@ -19,7 +19,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19
LayerBlockType, cdiv, is_pin_memory_available)
20
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21
FlashAttentionMetadata)
22
-from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
22
+from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
23
from vllm.v1.outputs import ModelRunnerOutput
24
from vllm.v1.sample.metadata import SamplingMetadata
25
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -82,12 +82,10 @@ class GPUModelRunner:
82
self.input_registry = INPUT_REGISTRY
83
self.mm_registry = MULTIMODAL_REGISTRY
84
85
- # NOTE: mm_input_mapper_client and mm_hasher are only used for memory
86
- # profiling.
87
- self.mm_input_mapper_client = MMInputMapperClient(self.model_config)
88
- self.mm_hasher = MMHasher()
89
- self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
90
- cache_config.enable_prefix_caching
85
+ # NOTE: Initialized input mapper is only used for processing dummy
86
+ # multimodal data into multimodal kwargs for GPU memory profiling.
87
+ self.mm_input_mapper_profiling = MMInputMapperClient(self.model_config)
88
+ self.mm_input_mapper_profiling.use_cache = False
89
90
self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
91
self.encoder_cache_size = self.scheduler_config.encoder_cache_size
@@ -722,8 +720,6 @@ class GPUModelRunner:
720
]
721
722
# Profile with multimodal encoder & encoder cache.
725
- # TODO (ywang96): generalize this beyond image modality since
726
- # mm_input_mapper only supports image inputs.
723
if self.is_multimodal_model:
724
725
# Create dummy batch of multimodal inputs.
@@ -735,15 +731,30 @@ class GPUModelRunner:
731
dummy_mm_data = dummy_request_data.multi_modal_data
732
733
# NOTE: Currently model is profiled with a single non-text
738
- # modality even when it supports multiple.
739
- max_tokens_per_mm_item = max(
740
- self.mm_registry.get_max_tokens_per_item_by_modality(
741
- self.model_config).values())
742
-
743
- max_num_mm_items_encoder_budget = min(
744
- self.max_num_encoder_input_tokens,
745
- self.encoder_cache_size) // max_tokens_per_mm_item
746
-
734
+ # modality with the max possible input tokens even when
735
+ # it supports multiple.
736
+ max_tokens_by_modality_dict = self.mm_registry.get_max_tokens_per_item_by_modality( # noqa: E501
737
+ self.model_config)
738
+
739
+ dummy_data_modality, max_tokens_per_mm_item = max(
740
+ max_tokens_by_modality_dict.items(), key=lambda item: item[1])
741
+
742
+ # Check how many items of this modality can be supported by
743
+ # the encoder cache budget.
744
+ encoder_cache_budget = min(self.max_num_encoder_input_tokens,
745
+ self.encoder_cache_size)
746
+ max_num_mm_items_encoder_budget = encoder_cache_budget // \
747
+ max_tokens_per_mm_item
748
+
749
+ # TODO: Allow users to set encoder_cache_budget in case this
750
+ # happens.
751
+ assert max_num_mm_items_encoder_budget > 0, (
752
+ f"Encoder cache budget={encoder_cache_budget} is too small to "
753
+ f"support the maximum possible size of multimodal embeddings"
754
+ f"={max_tokens_per_mm_item}.")
755
+
756
+ # Check how many items of this modality can be supported by
757
+ # the decoder budget.
758
max_mm_items_per_req = max(
759
self.mm_registry.get_mm_limits_per_prompt(
760
self.model_config).values())
@@ -763,33 +774,24 @@ class GPUModelRunner:
774
# they are scheduled to be processed separately.
775
776
# Case when models have a merged processor, their dummy data is
766
- # already batched `MultiModalKwargs`, therefore we need to "unbatch"
767
- # and take the first item in each batched tensor.
768
- # TODO (ywang96): This is somewhat hacky. Refactor this to be
769
- # consistent with the other case.
777
+ # already batched `MultiModalKwargs`, therefore we take the first
778
+ # `MultiModalKwargsItem` from the desired modality to profile on.
779
if isinstance(dummy_mm_data, MultiModalKwargs):
771
- dummy_mm_kwargs = {
772
- k: v[0].unsqueeze(0)
773
- for k, v in dummy_mm_data.items()
774
- }
780
+ dummy_mm_item = dummy_mm_data.get_item(
781
+ modality=dummy_data_modality, item_index=0)
782
+ dummy_mm_kwargs = MultiModalKwargs.from_items([dummy_mm_item])
783
784
# Case when models have dummy data explicitly defined as
785
# `MultiModalDataDict`, so they need to be processed through input
786
# mapper.
787
+ # TODO (ywang96): deprecate this path once merged processor is
788
+ # supported on all models.
789
else:
780
- # Compute MM hashes (if enabled)
781
- mm_hashes = None
782
- if self.use_hash:
783
- mm_hashes = self.mm_hasher.hash_dummy_mm_data(
784
- dummy_mm_data)
785
-
786
- mm_kwargs_list = self.mm_input_mapper_client.process_inputs(
790
+ mm_kwargs_list = self.mm_input_mapper_profiling.process_inputs(
791
mm_data=dummy_mm_data,
788
- mm_hashes=mm_hashes,
792
+ mm_hashes=None,
793
mm_processor_kwargs=None,
794
precomputed_mm_inputs=None)
791
-
792
- # Take the first `MultiModalKwargs`
795
dummy_mm_kwargs = mm_kwargs_list[0]
796
797
batched_dummy_mm_inputs = MultiModalKwargs.batch(