@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91b361ae89
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] Extend beyond image modality and support mixed-modality inference with Llava-OneVision (#11685)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Roger Wang committed Jan 6, 2025 at 11:58 UTC 91b361ae898c944f823534121613f9d3dc19d7d1
17 files changed +636 -282
docs/source/models/supported_models.md
+1 -1
@@ -647,7 +647,7 @@ See [this page](#generative-models) for more information on how to use generativ
647 - `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc.
648 -
649 - ✅︎
650 - -
650 + - ✅︎
651 * - `MiniCPMV`
652 - MiniCPM-V
653 - T + I<sup>E+</sup>
tests/multimodal/test_utils.py
+208 -1
@@ -2,16 +2,22 @@ import base64
2 import mimetypes
3 import os
4 from tempfile import NamedTemporaryFile, TemporaryDirectory
5 -from typing import Dict, Tuple
5 +from typing import TYPE_CHECKING, Dict, NamedTuple, Optional, Tuple
6
7 import numpy as np
8 import pytest
9 from PIL import Image, ImageChops
10 from transformers import AutoConfig, AutoTokenizer
11
12 +from vllm.multimodal.inputs import PlaceholderRange
13 from vllm.multimodal.utils import (MediaConnector,
14 + merge_and_sort_multimodal_metadata,
15 repeat_and_pad_placeholder_tokens)
16
17 +if TYPE_CHECKING:
18 + from vllm.multimodal.hasher import MultiModalHashDict
19 + from vllm.multimodal.inputs import MultiModalPlaceholderDict
20 +
21 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
22 TEST_IMAGE_URLS = [
23 "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
@@ -191,3 +197,204 @@ def test_repeat_and_pad_placeholder_tokens(model):
197 assert new_prompt == expected_prompt
198 assert new_token_ids == expected_token_ids
199 assert ranges == expected_ranges
200 +
201 +
202 +# Used for the next two tests related to `merge_and_sort_multimodal_metadata`.
203 +class TestCase(NamedTuple):
204 + mm_positions: "MultiModalPlaceholderDict"
205 + mm_hashes: Optional["MultiModalHashDict"]
206 + expected_modalities: list[str]
207 + expected_ranges: list[PlaceholderRange]
208 + expected_hashes: Optional[list[str]]
209 +
210 +
211 +def test_merge_and_sort_multimodal_metadata():
212 +
213 + test_cases = [
214 + # Single modality should return result as is but flattened
215 + TestCase(
216 + mm_positions={
217 + "image": [
218 + PlaceholderRange(offset=0, length=2),
219 + PlaceholderRange(offset=3, length=2),
220 + ]
221 + },
222 + mm_hashes={"image": ["hash1", "hash2"]},
223 + expected_modalities=["image"],
224 + expected_ranges=[
225 + PlaceholderRange(offset=0, length=2),
226 + PlaceholderRange(offset=3, length=2),
227 + ],
228 + expected_hashes=["hash1", "hash2"],
229 + ),
230 +
231 + # Single modality without hashes return None for mm hash.
232 + TestCase(
233 + mm_positions={
234 + "image": [
235 + PlaceholderRange(offset=0, length=2),
236 + PlaceholderRange(offset=2, length=2),
237 + ]
238 + },
239 + mm_hashes=None,
240 + expected_modalities=["image"],
241 + expected_ranges=[
242 + PlaceholderRange(offset=0, length=2),
243 + PlaceholderRange(offset=2, length=2),
244 + ],
245 + expected_hashes=None,
246 + ),
247 +
248 + # Multiple modalities with hashes should return sorted modalities
249 + # and flattened ranges and hashes.
250 + TestCase(
251 + mm_positions={
252 + "image": [
253 + PlaceholderRange(offset=7, length=4),
254 + PlaceholderRange(offset=11, length=5),
255 + ],
256 + "audio": [
257 + PlaceholderRange(offset=0, length=2),
258 + PlaceholderRange(offset=2, length=3),
259 + ]
260 + },
261 + mm_hashes={
262 + "image": ["image_hash1", "image_hash2"],
263 + "audio": ["audio_hash1", "audio_hash2"],
264 + },
265 + expected_modalities=["audio", "image"],
266 + expected_ranges=[
267 + PlaceholderRange(offset=0, length=2),
268 + PlaceholderRange(offset=2, length=3),
269 + PlaceholderRange(offset=7, length=4),
270 + PlaceholderRange(offset=11, length=5),
271 + ],
272 + expected_hashes=[
273 + "audio_hash1", "audio_hash2", "image_hash1", "image_hash2"
274 + ],
275 + ),
276 +
277 + # Multiple modalities without hashes should return sorted modalities
278 + # and flattened ranges and None.
279 + TestCase(
280 + mm_positions={
281 + "image": [
282 + PlaceholderRange(offset=7, length=4),
283 + PlaceholderRange(offset=11, length=5),
284 + ],
285 + "audio": [
286 + PlaceholderRange(offset=0, length=2),
287 + PlaceholderRange(offset=2, length=3),
288 + ]
289 + },
290 + mm_hashes=None,
291 + expected_modalities=["audio", "image"],
292 + expected_ranges=[
293 + PlaceholderRange(offset=0, length=2),
294 + PlaceholderRange(offset=2, length=3),
295 + PlaceholderRange(offset=7, length=4),
296 + PlaceholderRange(offset=11, length=5),
297 + ],
298 + expected_hashes=None,
299 + ),
300 +
301 + # Three modalities
302 + TestCase(
303 + mm_positions={
304 + "image": [
305 + PlaceholderRange(offset=15, length=7),
306 + PlaceholderRange(offset=22, length=8),
307 + ],
308 + "audio": [
309 + PlaceholderRange(offset=0, length=2),
310 + ],
311 + "video": [
312 + PlaceholderRange(offset=3, length=4),
313 + PlaceholderRange(offset=7, length=5),
314 + PlaceholderRange(offset=12, length=6),
315 + ]
316 + },
317 + mm_hashes={
318 + "image": ["image_hash1", "image_hash2"],
319 + "audio": ["audio_hash1"],
320 + "video": ["video_hash1", "video_hash2", "video_hash3"]
321 + },
322 + expected_modalities=["audio", "video", "image"],
323 + expected_ranges=[
324 + PlaceholderRange(offset=0, length=2),
325 + PlaceholderRange(offset=3, length=4),
326 + PlaceholderRange(offset=7, length=5),
327 + PlaceholderRange(offset=12, length=6),
328 + PlaceholderRange(offset=15, length=7),
329 + PlaceholderRange(offset=22, length=8),
330 + ],
331 + expected_hashes=[
332 + "audio_hash1", "video_hash1", "video_hash2", "video_hash3",
333 + "image_hash1", "image_hash2"
334 + ],
335 + ),
336 + ]
337 +
338 + for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
339 + expected_hashes) in test_cases:
340 + modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
341 + mm_positions, mm_hashes)
342 +
343 + assert modalities == expected_modalities
344 + assert ranges == expected_ranges
345 + assert hashes == expected_hashes
346 +
347 +
348 +def test_merge_and_sort_multimodal_metadata_with_interleaving():
349 +
350 + test_cases = [
351 +
352 + # <image> <audio> <image> <audio>
353 + TestCase(
354 + mm_positions={
355 + "image": [
356 + PlaceholderRange(offset=0, length=4),
357 + PlaceholderRange(offset=8, length=2),
358 + ],
359 + "audio": [
360 + PlaceholderRange(offset=5, length=2),
361 + PlaceholderRange(offset=11, length=4),
362 + ]
363 + },
364 + mm_hashes={
365 + "image": ["image_hash1", "image_hash2"],
366 + "audio": ["audio_hash1", "audio_hash2"],
367 + },
368 + expected_modalities=[],
369 + expected_ranges=[],
370 + expected_hashes=None,
371 + ),
372 +
373 + # <image> <image> <video> <audio> <image>
374 + TestCase(
375 + mm_positions={
376 + "image": [
377 + PlaceholderRange(offset=0, length=2),
378 + PlaceholderRange(offset=2, length=3),
379 + PlaceholderRange(offset=20, length=4),
380 + ],
381 + "audio": [
382 + PlaceholderRange(offset=5, length=2),
383 + ],
384 + "video": [
385 + PlaceholderRange(offset=8, length=5),
386 + ]
387 + },
388 + mm_hashes=None,
389 + expected_modalities=[],
390 + expected_ranges=[],
391 + expected_hashes=None,
392 + ),
393 + ]
394 +
395 + for case in test_cases:
396 + with pytest.raises(ValueError) as ex_info:
397 + merge_and_sort_multimodal_metadata(case.mm_positions,
398 + case.mm_hashes)
399 +
400 + assert "Interleaved mixed-modality" in str(ex_info.value)
tests/v1/core/test_kv_cache_utils.py
+11 -7
@@ -1,6 +1,6 @@
1 import pytest
2
3 -from vllm.inputs import token_inputs
3 +from vllm.multimodal.inputs import MultiModalKwargs
4 from vllm.sampling_params import SamplingParams
5 from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
6 KVCacheBlock,
@@ -14,14 +14,18 @@ def make_request(request_id,
14 prompt_token_ids,
15 mm_positions=None,
16 mm_hashes=None):
17 + if mm_positions is None:
18 + multi_modal_inputs = None
19 + else:
20 + multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
21 +
22 return Request(
23 request_id=request_id,
19 - inputs=token_inputs(
20 - prompt_token_ids=prompt_token_ids,
21 - multi_modal_placeholders={"image": mm_positions}
22 - if mm_positions else None,
23 - multi_modal_hashes=mm_hashes,
24 - ),
24 + prompt=None,
25 + prompt_token_ids=prompt_token_ids,
26 + multi_modal_inputs=multi_modal_inputs,
27 + multi_modal_hashes=mm_hashes,
28 + multi_modal_placeholders=mm_positions,
29 sampling_params=SamplingParams(max_tokens=17),
30 eos_token_id=100,
31 arrival_time=0,
tests/v1/core/test_prefix_caching.py
+11 -6
@@ -1,8 +1,7 @@
1 """Compare the with and without prefix caching."""
2 import pytest
3
4 -from vllm.inputs import token_inputs
5 -from vllm.multimodal.inputs import PlaceholderRange
4 +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
5 from vllm.sampling_params import SamplingParams
6 from vllm.utils import cdiv
7 from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
@@ -13,12 +12,18 @@ def make_request(request_id,
12 prompt_token_ids,
13 mm_positions=None,
14 mm_hashes=None):
15 + if mm_positions is None:
16 + multi_modal_inputs = None
17 + else:
18 + multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
19 +
20 return Request(
21 request_id=request_id,
18 - inputs=token_inputs(prompt_token_ids=prompt_token_ids,
19 - multi_modal_placeholders={"image": mm_positions}
20 - if mm_positions else None,
21 - multi_modal_hashes=mm_hashes),
22 + prompt=None,
23 + prompt_token_ids=prompt_token_ids,
24 + multi_modal_inputs=multi_modal_inputs,
25 + multi_modal_hashes=mm_hashes,
26 + multi_modal_placeholders=mm_positions,
27 sampling_params=SamplingParams(max_tokens=17),
28 eos_token_id=100,
29 arrival_time=0,
vllm/model_executor/models/interfaces.py
+5 -1
@@ -39,8 +39,12 @@ class SupportsMultiModal(Protocol):
39
40 The output embeddings must be one of the following formats:
41 - A list or tuple of 2D tensors, where each tensor corresponds to
42 - each input image.
42 + each input multimodal data item (e.g, image).
43 - A single 3D tensor, with the batch dimension grouping the 2D tensors.
44 +
45 + NOTE: The returned multimodal embeddings must be in the same order as
46 + the appearances of their corresponding multimodal data item in the
47 + input prompt.
48 """
49 ...
50
vllm/model_executor/models/llava_onevision.py
+33 -32
@@ -35,6 +35,9 @@ from .siglip import SiglipVisionModel
35 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36 maybe_prefix, merge_multimodal_embeddings)
37
38 +# For profile run
39 +_MAX_FRAMES_PER_VIDEO = 16
40 +
41
42 class LlavaOnevisionVideoPixelInputs(TypedDict):
43 type: Literal["pixel_values_videos"]
@@ -223,8 +226,10 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
226 max_image_tokens = self._get_max_image_tokens() * max_images
227 max_total_frames = self._get_max_video_frames(seq_len -
228 max_image_tokens)
229 + max_frames_per_video = min(max_total_frames // max(max_videos, 1),
230 + _MAX_FRAMES_PER_VIDEO)
231
227 - return max(max_total_frames // max(max_videos, 1), 1)
232 + return max(max_frames_per_video, 1)
233
234 def _get_max_video_tokens(self, seq_len: int) -> int:
235 target_width, target_height = self._get_image_size_with_most_features()
@@ -558,13 +563,15 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
563 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
564 modalities = {}
565
561 - if "pixel_values" in kwargs:
562 - modalities["images"] = self._parse_and_validate_image_input(
563 - **kwargs)
564 -
565 - if "pixel_values_videos" in kwargs:
566 - modalities["videos"] = self._parse_and_validate_video_input(
567 - **kwargs)
566 + # Preserve the order of modalities if there are multiple of them
567 + # from the order of kwargs.
568 + for input_key in kwargs:
569 + if input_key == "pixel_values" and "images" not in modalities:
570 + modalities["images"] = self._parse_and_validate_image_input(
571 + **kwargs)
572 + if input_key == "pixel_values_videos" and "videos" not in modalities: # noqa E501
573 + modalities["videos"] = self._parse_and_validate_video_input(
574 + **kwargs)
575
576 return modalities
577
@@ -824,21 +831,21 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
831 if not modalities:
832 return None
833
827 - # We make a tuple of each embedding with its modality string. This is a
828 - # temporary workaround for models to handle mixed modalities when
829 - # get_multimodal_embeddings and get_input_embeddings are called
830 - # separately.
831 - # TODO(ywang96): Add support for mixed-modality inference for v1.
832 - multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
833 -
834 - if "images" in modalities:
835 - image_input = modalities["images"]
836 - vision_embeddings = self._process_image_input(image_input)
837 - multimodal_embeddings.append((vision_embeddings, "image"))
838 - if "videos" in modalities:
839 - video_input = modalities["videos"]
840 - video_embeddings = self._process_video_pixels(video_input)
841 - multimodal_embeddings.append((video_embeddings, "video"))
834 + # The result multimodal_embeddings is tuple of tensors, with each
835 + # tensor correspoending to a multimodal data item (image or video).
836 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
837 +
838 + # NOTE: It is important to iterate over the keys in this dictionary
839 + # to preserve the order of the modalities.
840 + for modality in modalities:
841 + if modality == "images":
842 + image_input = modalities["images"]
843 + vision_embeddings = self._process_image_input(image_input)
844 + multimodal_embeddings += tuple(vision_embeddings)
845 + if modality == "videos":
846 + video_input = modalities["videos"]
847 + video_embeddings = self._process_video_pixels(video_input)
848 + multimodal_embeddings += tuple(video_embeddings)
849
850 return multimodal_embeddings
851
@@ -850,15 +857,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
857 ) -> torch.Tensor:
858 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
859 if multimodal_embeddings is not None:
853 - for embeddings, modality in multimodal_embeddings:
854 - if modality == "image":
855 - inputs_embeds = merge_multimodal_embeddings(
856 - input_ids, inputs_embeds, embeddings,
857 - self.config.image_token_index)
858 - if modality == "video":
859 - inputs_embeds = merge_multimodal_embeddings(
860 - input_ids, inputs_embeds, embeddings,
861 - self.config.video_token_index)
860 + inputs_embeds = merge_multimodal_embeddings(
861 + input_ids, inputs_embeds, multimodal_embeddings,
862 + [self.config.image_token_index, self.config.video_token_index])
863 return inputs_embeds
864
865 def forward(
vllm/model_executor/models/molmo.py
-3
@@ -972,8 +972,6 @@ def image_input_mapper_for_molmo(
972 assert len(data) == 1, "Molmo supports only one image per prompt."
973 data = data[0]
974
975 - # Remove unused dummy PIL image
976 - data.pop('raw_mm_data', None)
975 return MultiModalKwargs(data)
976
977
@@ -1019,7 +1017,6 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
1017 dummy_imgdata = {
1018 "images": out["images"],
1019 "image_input_idx": out["image_input_idx"],
1022 - "raw_mm_data": dummy_image,
1020 }
1021 if "image_masks" in out:
1022 dummy_imgdata["image_masks"] = out["image_masks"]
vllm/multimodal/__init__.py
+3
@@ -1,4 +1,5 @@
1 from .base import MultiModalPlaceholderMap, MultiModalPlugin
2 +from .hasher import MultiModalHashDict, MultiModalHasher
3 from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
4 MultiModalDataDict, MultiModalKwargs,
5 MultiModalPlaceholderDict, NestedTensors)
@@ -18,6 +19,8 @@ __all__ = [
19 "ModalityData",
20 "MultiModalDataBuiltins",
21 "MultiModalDataDict",
22 + "MultiModalHashDict",
23 + "MultiModalHasher",
24 "MultiModalKwargs",
25 "MultiModalPlaceholderDict",
26 "MultiModalPlaceholderMap",
vllm/multimodal/hasher.py new
+100
@@ -0,0 +1,100 @@
1 +import pickle
2 +from typing import TYPE_CHECKING, Iterable, Mapping, Optional
3 +
4 +import numpy as np
5 +import torch
6 +from blake3 import blake3
7 +from PIL import Image
8 +
9 +from vllm.logger import init_logger
10 +
11 +if TYPE_CHECKING:
12 + from vllm.inputs import TokensPrompt
13 +
14 +logger = init_logger(__name__)
15 +
16 +MultiModalHashDict = Mapping[str, list[str]]
17 +"""
18 +A dictionary containing hashes for items in each modality.
19 +"""
20 +
21 +
22 +class MultiModalHasher:
23 +
24 + @classmethod
25 + def serialize_item(cls, obj: object) -> bytes:
26 + # Simple cases
27 + if isinstance(obj, str):
28 + return obj.encode("utf-8")
29 + if isinstance(obj, bytes):
30 + return obj
31 + if isinstance(obj, Image.Image):
32 + return obj.tobytes()
33 +
34 + # Convertible to NumPy arrays
35 + if isinstance(obj, torch.Tensor):
36 + obj = obj.numpy()
37 + if isinstance(obj, (int, float)):
38 + obj = np.array(obj)
39 + if isinstance(obj, np.ndarray):
40 + return obj.tobytes()
41 +
42 + logger.warning(
43 + "No serialization method found for %s. "
44 + "Falling back to pickle.", type(obj))
45 +
46 + return pickle.dumps(obj)
47 +
48 + @classmethod
49 + def item_to_bytes(
50 + cls,
51 + key: str,
52 + obj: object,
53 + ) -> Iterable[tuple[bytes, bytes]]:
54 + # Recursive cases
55 + if isinstance(obj, (list, tuple)):
56 + for i, elem in enumerate(obj):
57 + yield from cls.item_to_bytes(f"{key}.{i}", elem)
58 + elif isinstance(obj, dict):
59 + for k, v in obj.items():
60 + yield from cls.item_to_bytes(f"{key}.{k}", v)
61 + else:
62 + key_bytes = cls.serialize_item(key)
63 + value_bytes = cls.serialize_item(obj)
64 + yield key_bytes, value_bytes
65 +
66 + @classmethod
67 + def hash_kwargs(cls, **kwargs: object) -> str:
68 + hasher = blake3()
69 +
70 + for k, v in kwargs.items():
71 + for k_bytes, v_bytes in cls.item_to_bytes(k, v):
72 + hasher.update(k_bytes)
73 + hasher.update(v_bytes)
74 +
75 + return hasher.hexdigest()
76 +
77 + @classmethod
78 + def hash_prompt_mm_data(
79 + cls, prompt: "TokensPrompt") -> Optional["MultiModalHashDict"]:
80 + """Hash multimodal data in the user input prompt if they exist."""
81 +
82 + if "multi_modal_data" not in prompt:
83 + return None
84 +
85 + mm_data = prompt["multi_modal_data"]
86 + if not mm_data:
87 + # mm_data can be None or an empty dict.
88 + return None
89 +
90 + mm_items = {
91 + modality: items if isinstance(items, list) else [items]
92 + for modality, items in mm_data.items()
93 + }
94 +
95 + mm_hashes = {
96 + modality: [cls.hash_kwargs(**{modality: item}) for item in items]
97 + for modality, items in mm_items.items()
98 + }
99 +
100 + return mm_hashes
vllm/multimodal/inputs.py
+6 -3
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2 from collections import UserDict, defaultdict
3 from collections.abc import Mapping, Sequence
4 from dataclasses import dataclass
5 -from typing import (Any, Literal, Optional, TypedDict, TypeVar, Union, cast,
6 - final)
5 +from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
6 + Union, cast, final)
7
8 import numpy as np
9 import torch
@@ -14,6 +14,9 @@ from typing_extensions import NotRequired, TypeAlias
14
15 from vllm.utils import JSONTree, full_groupby, is_list_of, json_map_leaves
16
17 +if TYPE_CHECKING:
18 + from .hasher import MultiModalHashDict
19 +
20 _T = TypeVar("_T")
21
22 HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
@@ -513,7 +516,7 @@ class MultiModalInputsV2(TypedDict):
516 mm_kwargs: MultiModalKwargs
517 """Keyword arguments to be directly passed to the model after batching."""
518
516 - mm_hashes: NotRequired[list[str]]
519 + mm_hashes: NotRequired[Optional["MultiModalHashDict"]]
520 """The hashes of the multi-modal data."""
521
522 mm_placeholders: MultiModalPlaceholderDict
vllm/multimodal/processing.py
+30 -62
@@ -1,4 +1,3 @@
1 -import pickle
1 import re
2 from abc import ABC, abstractmethod
3 from collections import defaultdict
@@ -7,18 +6,16 @@ from dataclasses import dataclass, field
6 from functools import lru_cache
7 from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
10 -import numpy as np
11 -import torch
12 -from blake3 import blake3
13 -from PIL import Image
9 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
10
11 +from vllm import envs
12 from vllm.inputs import DummyData, InputProcessingContext
13 from vllm.logger import init_logger
14 from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
15 encode_tokens)
16 from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
17
18 +from .hasher import MultiModalHasher
19 from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
20 MultiModalInputsV2, MultiModalKwargs,
21 MultiModalKwargsItem, PlaceholderRange)
@@ -486,56 +483,6 @@ class ProcessingCache:
483 logger.debug("ProcessingCache: hit_ratio = %.2f",
484 cache_stats.hit_ratio)
485
489 - def _serialize_item(self, obj: object) -> bytes:
490 - # Simple cases
491 - if isinstance(obj, str):
492 - return obj.encode("utf-8")
493 - if isinstance(obj, bytes):
494 - return obj
495 - if isinstance(obj, Image.Image):
496 - return obj.tobytes()
497 -
498 - # Convertible to NumPy arrays
499 - if isinstance(obj, torch.Tensor):
500 - obj = obj.numpy()
501 - if isinstance(obj, (int, float)):
502 - obj = np.array(obj)
503 - if isinstance(obj, np.ndarray):
504 - return obj.tobytes()
505 -
506 - logger.warning(
507 - "No serialization method found for %s. "
508 - "Falling back to pickle.", type(obj))
509 -
510 - return pickle.dumps(obj)
511 -
512 - def _item_to_bytes(
513 - self,
514 - key: str,
515 - obj: object,
516 - ) -> Iterable[tuple[bytes, bytes]]:
517 - # Recursive cases
518 - if isinstance(obj, (list, tuple)):
519 - for i, elem in enumerate(obj):
520 - yield from self._item_to_bytes(f"{key}.{i}", elem)
521 - elif isinstance(obj, dict):
522 - for k, v in obj.items():
523 - yield from self._item_to_bytes(f"{key}.{k}", v)
524 - else:
525 - key_bytes = self._serialize_item(key)
526 - value_bytes = self._serialize_item(obj)
527 - yield key_bytes, value_bytes
528 -
529 - def _hash_kwargs(self, **kwargs: object) -> str:
530 - hasher = blake3()
531 -
532 - for k, v in kwargs.items():
533 - for k_bytes, v_bytes in self._item_to_bytes(k, v):
534 - hasher.update(k_bytes)
535 - hasher.update(v_bytes)
536 -
537 - return hasher.hexdigest()
538 -
486 def get(
487 self,
488 model_id: str,
@@ -554,9 +501,9 @@ class ProcessingCache:
501 """
502 self._maybe_log_cache_stats()
503
557 - cache_key = self._hash_kwargs(model_id=model_id,
558 - **{modality: input_item},
559 - **input_kwargs)
504 + cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
505 + **{modality: input_item},
506 + **input_kwargs)
507 return self._cache.get(cache_key)
508
509 def put(
@@ -571,9 +518,9 @@ class ProcessingCache:
518 Put a processed multi-modal item into the cache
519 according to its dependencies (see :meth:`get`).
520 """
574 - cache_key = self._hash_kwargs(model_id=model_id,
575 - **{modality: input_item},
576 - **input_kwargs)
521 + cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
522 + **{modality: input_item},
523 + **input_kwargs)
524 self._cache.put(cache_key, output_kwargs)
525
526
@@ -1049,6 +996,24 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
996 """
997 mm_items = self._to_mm_items(mm_data)
998
999 + # Create MM hashes (only used in V1)
1000 + # TODO: Use these hash keys for caching operations in apply_hf_processor
1001 + # instead of rehashing.
1002 +
1003 + if envs.VLLM_USE_V1:
1004 + model_id = self.ctx.model_config.model
1005 + mm_hashes = {
1006 + modality: [
1007 + MultiModalHasher.hash_kwargs(model_id=model_id,
1008 + **{modality: item},
1009 + **hf_processor_mm_kwargs)
1010 + for item in items
1011 + ]
1012 + for modality, items in mm_items.items()
1013 + }
1014 + else:
1015 + mm_hashes = None
1016 +
1017 prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
1018 prompt_text,
1019 mm_items,
@@ -1122,6 +1087,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1087 prompt=prompt_text,
1088 prompt_token_ids=prompt_ids,
1089 mm_kwargs=mm_kwargs,
1090 + mm_hashes=mm_hashes,
1091 mm_placeholders=mm_placeholder_ranges,
1092 )
1093
@@ -1174,7 +1140,9 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1140 "tokens.")
1141
1142 total_len = len(prompt_token_ids)
1177 - if total_len > seq_len:
1143 +
1144 + # V0 does not support chunked prefill.
1145 + if total_len > seq_len and not envs.VLLM_USE_V1:
1146 logger.warning(
1147 "The context length (%d) of the model is too short "
1148 "to hold the multi-modal embeddings in the worst case "
vllm/multimodal/utils.py
+85 -1
@@ -1,6 +1,6 @@
1 from functools import lru_cache
2 from pathlib import Path
3 -from typing import Optional, TypeVar, Union
3 +from typing import TYPE_CHECKING, Optional, TypeVar, Union
4 from urllib.parse import ParseResult, urlparse
5
6 import numpy as np
@@ -25,6 +25,10 @@ cached_get_tokenizer = lru_cache(get_tokenizer)
25
26 _M = TypeVar("_M")
27
28 +if TYPE_CHECKING:
29 + from .hasher import MultiModalHashDict
30 + from .inputs import MultiModalPlaceholderDict
31 +
32
33 class MediaConnector:
34
@@ -437,3 +441,83 @@ def consecutive_placeholder_ranges(
441 PlaceholderRange(offset=initial_offset + i * item_size,
442 length=item_size) for i in range(num_items)
443 ]
444 +
445 +
446 +def merge_and_sort_multimodal_metadata(
447 + mm_positions: "MultiModalPlaceholderDict",
448 + mm_hashes: Optional["MultiModalHashDict"],
449 +) -> tuple[list[str], list[PlaceholderRange], Optional[list[str]]]:
450 + """Given a MultiModalPlaceholderDict, merge all PlaceholderRange
451 + objects from all available modalities into a single list of
452 + PlaceholderRange, sorted by their offset (starting index in the input
453 + sequence) in the ascending order.
454 +
455 + Optionally if a MultiModalHashDict is given, same operation will be
456 + applied to the object and the sorted list of hashes will be returned.
457 +
458 + Raises:
459 + ValueError: If the input prompt has interleaved placeholders from
460 + different modalities (e.g, "<image><audio><image> Describe the
461 + content.")
462 +
463 + Returns:
464 + list[str]: Sorted list of involved modalities.
465 + list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
466 + mm_positions.
467 + Optional[list[str]]: Sorted list of all hashes from mm_hashes if
468 + given, None otherwise.
469 + """
470 +
471 + modalities = list(mm_positions.keys())
472 +
473 + assert len(modalities) > 0, "No modalities found in the mm_positions."
474 +
475 + # For single modality, placeholder ranges and hashes are already sorted
476 + # so we can return the list directly.
477 + if len(modalities) == 1:
478 + if mm_hashes is None:
479 + return modalities, list(mm_positions[modalities[0]]), None
480 + else:
481 + return modalities, list(mm_positions[modalities[0]]), list(
482 + mm_hashes[modalities[0]])
483 +
484 + placeholder_lists_with_modality = [(modality, mm_positions[modality])
485 + for modality in modalities]
486 +
487 + if mm_hashes is None:
488 + sorted_placeholder_lists = sorted(placeholder_lists_with_modality,
489 + key=lambda x: x[1][0]['offset'])
490 + sorted_hash_lists = None
491 + else:
492 + hashes_lists = [
493 + mm_hashes[modality] for modality in modalities
494 + if modality in mm_hashes
495 + ]
496 + sorted_pairs = sorted(zip(placeholder_lists_with_modality,
497 + hashes_lists),
498 + key=lambda x: x[0][1][0]['offset'])
499 + sorted_placeholder_tuple, sorted_hash_tuple = zip(*sorted_pairs)
500 + sorted_placeholder_lists = list(sorted_placeholder_tuple)
501 + sorted_hash_lists = list(sorted_hash_tuple)
502 +
503 + sorted_modalities = [modality for modality, _ in sorted_placeholder_lists]
504 +
505 + # Flatten sorted list of lists to a single list and verify there is no
506 + # interleaving of placeholders from different modalities.
507 + merged_placeholders: list[PlaceholderRange] = []
508 + for modality, placeholder_list in sorted_placeholder_lists:
509 + if merged_placeholders and placeholder_list[0][
510 + 'offset'] < merged_placeholders[-1]['offset']:
511 + raise ValueError(
512 + "Interleaved mixed-modality inference is currently not "
513 + "supported.")
514 + merged_placeholders.extend(placeholder_list)
515 +
516 + if sorted_hash_lists is not None:
517 + merged_hashes = []
518 + for hash_list in sorted_hash_lists:
519 + merged_hashes.extend(hash_list)
520 + else:
521 + merged_hashes = None
522 +
523 + return sorted_modalities, merged_placeholders, merged_hashes
vllm/v1/engine/__init__.py
+10 -8
@@ -1,12 +1,14 @@
1 import enum
2 from dataclasses import dataclass
3 -from typing import List, Optional, Union
3 +from typing import TYPE_CHECKING, List, Optional, Union
4
5 import msgspec
6
7 -from vllm.lora.request import LoRARequest
8 -from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9 -from vllm.sampling_params import SamplingParams
7 +if TYPE_CHECKING:
8 + from vllm.lora.request import LoRARequest
9 + from vllm.multimodal import MultiModalKwargs
10 + from vllm.multimodal.inputs import PlaceholderRange
11 + from vllm.sampling_params import SamplingParams
12
13
14 @dataclass
@@ -21,13 +23,13 @@ class EngineCoreRequest:
23 # always be tokenized?
24 prompt: Optional[str]
25 prompt_token_ids: List[int]
24 - mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
26 + mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
27 mm_hashes: Optional[List[str]]
26 - mm_placeholders: Optional[MultiModalPlaceholderDict]
27 - sampling_params: SamplingParams
28 + mm_placeholders: Optional[List["PlaceholderRange"]]
29 + sampling_params: "SamplingParams"
30 eos_token_id: Optional[int]
31 arrival_time: float
30 - lora_request: Optional[LoRARequest]
32 + lora_request: Optional["LoRARequest"]
33
34
35 class EngineCoreOutput(
vllm/v1/engine/mm_input_mapper.py
-67
@@ -1,10 +1,6 @@
1 from typing import Any, Dict, List, Optional
2
3 -import PIL
4 -from blake3 import blake3
5 -
3 from vllm.config import ModelConfig
7 -from vllm.inputs import PromptType
4 from vllm.logger import init_logger
5 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
6 MultiModalKwargs, MultiModalRegistry)
@@ -144,66 +140,3 @@ class MMInputMapperServer:
140 full_mm_inputs.append(mm_input)
141
142 return full_mm_inputs
147 -
148 -
149 -class MMHasher:
150 -
151 - def __init__(self):
152 - pass
153 -
154 - def hash_dummy_mm_data(
155 - self,
156 - mm_data: Optional[MultiModalDataDict]) -> Optional[List[str]]:
157 - """Hash user-defined dummy multimodal data used for profiling."""
158 -
159 - if mm_data is None:
160 - return None
161 -
162 - image_inputs = mm_data['image']
163 -
164 - # This is a temporary workaround for models (e.g, Molmo) that
165 - # process multimodal data in the input processor (therefore
166 - # image_inputs is MultiModalKwargs instead of raw input format).
167 - # `raw_mm_data` with the original input format is expected
168 - # in this case.
169 - if isinstance(image_inputs, dict):
170 - assert "raw_mm_data" in image_inputs and isinstance(
171 - image_inputs["raw_mm_data"], PIL.Image.Image)
172 - image_inputs = image_inputs.pop("raw_mm_data")
173 -
174 - return self.hash_images(image_inputs)
175 -
176 - def hash_prompt_mm_data(self, prompt: PromptType) -> Optional[List[str]]:
177 - """Hash multimodal data in the user input prompt if they exist."""
178 -
179 - if "multi_modal_data" not in prompt:
180 - return None
181 -
182 - mm_data = prompt["multi_modal_data"]
183 - if not mm_data:
184 - # mm_data can be None or an empty dict.
185 - return None
186 -
187 - image_inputs = mm_data["image"]
188 -
189 - return self.hash_images(image_inputs)
190 -
191 - def hash_images(self, image_inputs) -> Optional[List[str]]:
192 - """Hash PIL image objects to strings."""
193 - if not isinstance(image_inputs, list):
194 - image_inputs = [image_inputs]
195 - assert len(image_inputs) > 0
196 -
197 - ret = []
198 - for image in image_inputs:
199 - assert isinstance(image, PIL.Image.Image)
200 -
201 - # Convert image to bytes
202 - bytes = image.tobytes()
203 -
204 - # Hash image bytes
205 - hasher = blake3()
206 - hasher.update(bytes)
207 - ret.append(hasher.hexdigest())
208 -
209 - return ret
vllm/v1/engine/processor.py
+74 -27
@@ -7,14 +7,15 @@ from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
7 from vllm.inputs.parse import is_encoder_decoder_inputs
8 from vllm.inputs.preprocess import InputPreprocessor
9 from vllm.lora.request import LoRARequest
10 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
11 - MultiModalRegistry)
10 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalHasher,
11 + MultiModalKwargs, MultiModalRegistry)
12 +from vllm.multimodal.utils import merge_and_sort_multimodal_metadata
13 from vllm.pooling_params import PoolingParams
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sampling_params import SamplingParams
16 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
17 from vllm.v1.engine import EngineCoreRequest
17 -from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
18 +from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
19
20
21 class Processor:
@@ -47,7 +48,6 @@ class Processor:
48 # Multi-modal hasher (for images)
49 self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
50 cache_config.enable_prefix_caching
50 - self.mm_hasher = MMHasher()
51
52 def process_inputs(
53 self,
@@ -73,11 +73,6 @@ class Processor:
73 assert priority == 0, "vLLM V1 does not support priority at the moment."
74 assert trace_headers is None, "vLLM V1 does not support tracing yet."
75
76 - # Compute MM hashes (if enabled)
77 - mm_hashes = None
78 - if self.use_hash:
79 - mm_hashes = self.mm_hasher.hash_prompt_mm_data(prompt)
80 -
76 # Process inputs.
77 preprocessed_inputs = self.input_preprocessor.preprocess(
78 prompt,
@@ -108,8 +103,20 @@ class Processor:
103 sampling_params.update_from_generation_config(
104 self.generation_config_fields, eos_token_id)
105
106 + # Multimodal related.
107 + # Compute MM hashes (if enabled)
108 + mm_hashes = None
109 + if self.use_hash:
110 + # Use mm_hashes from processed inputs if the model has merged
111 + # input processor.
112 + if decoder_inputs.multi_modal_hashes:
113 + mm_hashes = decoder_inputs.multi_modal_hashes
114 + # Fallback to using MultiModalHasher directly.
115 + else:
116 + mm_hashes = MultiModalHasher.hash_prompt_mm_data(prompt)
117 +
118 # For merged preprocessor, mm_data is already mm_inputs
112 - precomputed_mm_inputs = None
119 + precomputed_mm_inputs: Optional[list[MultiModalKwargs]] = None
120 decoder_mm_data = decoder_inputs.multi_modal_data
121 if isinstance(decoder_mm_data, MultiModalKwargs):
122 # The output of merged multi-modal processor (`decoder_mm_data`)
@@ -122,27 +129,67 @@ class Processor:
129 for item in decoder_mm_data.get_items(modality)
130 ]
131
125 - # Apply MM mapper
126 - mm_inputs = None
127 - if len(decoder_mm_data) > 0:
128 - mm_inputs = self.mm_input_mapper_client.process_inputs(
129 - decoder_mm_data,
132 + mm_positions = decoder_inputs.multi_modal_placeholders
133 +
134 + # Last-mile processing of multimodal metadata and inputs.
135 + if mm_positions:
136 +
137 + # Merge and flatten multimodal placeholders, hashes and inputs
138 + # from dictionaries to lists, and sort them by each item's position
139 + # in the input sequence.
140 + # NOTE: interleaved modalities are not supported.
141 + (
142 + sorted_modalities,
143 + sorted_mm_positions,
144 + sorted_mm_hashes,
145 + ) = merge_and_sort_multimodal_metadata(
146 + mm_positions,
147 mm_hashes,
131 - decoder_inputs.mm_processor_kwargs,
132 - precomputed_mm_inputs,
148 )
149
150 + # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple
151 + # modalities involved AND the model supports merged input processor.
152 + if len(sorted_modalities) > 1 and precomputed_mm_inputs:
153 +
154 + modality_order_dict = {
155 + modality: order
156 + for order, modality in enumerate(sorted_modalities)
157 + }
158 +
159 + # Sanity check to make sure each multimodal input has only one
160 + # modality key.
161 + for mm_input in precomputed_mm_inputs:
162 + assert len(mm_input.modalities) == 1
163 +
164 + # Sort MultiModalKwags to match sorted_mm_positions
165 + precomputed_mm_inputs = sorted(
166 + precomputed_mm_inputs,
167 + key=lambda mm_input: modality_order_dict[list(
168 + mm_input.modalities)[0]])
169 +
170 + # Apply mm input cache update (and input mapper if necessary).
171 + sorted_mm_inputs = self.mm_input_mapper_client.process_inputs(
172 + mm_data=decoder_mm_data,
173 + mm_hashes=sorted_mm_hashes,
174 + mm_processor_kwargs=decoder_inputs.mm_processor_kwargs,
175 + precomputed_mm_inputs=precomputed_mm_inputs,
176 + )
177 + else:
178 + sorted_mm_inputs = None
179 + sorted_mm_hashes = None
180 + sorted_mm_positions = None
181 +
182 return EngineCoreRequest(
136 - request_id,
137 - decoder_inputs.prompt,
138 - decoder_inputs.prompt_token_ids,
139 - mm_inputs,
140 - mm_hashes,
141 - decoder_inputs.multi_modal_placeholders,
142 - sampling_params,
143 - eos_token_id,
144 - arrival_time,
145 - lora_request,
183 + request_id=request_id,
184 + prompt=decoder_inputs.prompt,
185 + prompt_token_ids=decoder_inputs.prompt_token_ids,
186 + mm_inputs=sorted_mm_inputs,
187 + mm_hashes=sorted_mm_hashes,
188 + mm_placeholders=sorted_mm_positions,
189 + sampling_params=sampling_params,
190 + eos_token_id=eos_token_id,
191 + arrival_time=arrival_time,
192 + lora_request=lora_request,
193 )
194
195 def _validate_model_inputs(self, inputs: ProcessorInputs):
vllm/v1/request.py
+21 -27
@@ -1,15 +1,15 @@
1 import enum
2 from typing import TYPE_CHECKING, List, Optional, Union
3
4 -from vllm.inputs import DecoderOnlyInputs, SingletonInputsAdapter, token_inputs
4 from vllm.lora.request import LoRARequest
6 -from vllm.multimodal import MultiModalKwargs
5 from vllm.sampling_params import SamplingParams
6 from vllm.sequence import RequestMetrics
7 from vllm.v1.engine import EngineCoreRequest
8 from vllm.v1.utils import ConstantList
9
10 if TYPE_CHECKING:
11 + from vllm.multimodal import MultiModalKwargs
12 + from vllm.multimodal.inputs import PlaceholderRange
13 from vllm.v1.core.kv_cache_utils import BlockHashType
14
15
@@ -18,14 +18,17 @@ class Request:
18 def __init__(
19 self,
20 request_id: str,
21 - inputs: DecoderOnlyInputs,
21 + prompt: Optional[str],
22 + prompt_token_ids: List[int],
23 + multi_modal_inputs: Optional[List["MultiModalKwargs"]],
24 + multi_modal_hashes: Optional[List[str]],
25 + multi_modal_placeholders: Optional[List["PlaceholderRange"]],
26 sampling_params: SamplingParams,
27 eos_token_id: Optional[int],
28 arrival_time: float,
29 lora_request: Optional[LoRARequest] = None,
30 ) -> None:
31 self.request_id = request_id
28 - self.inputs = SingletonInputsAdapter(inputs)
32 self.sampling_params = sampling_params
33 # Because of LoRA, the eos token id can be different for each request.
34 self.eos_token_id = eos_token_id
@@ -41,26 +44,21 @@ class Request:
44 assert sampling_params.max_tokens is not None
45 self.max_tokens = sampling_params.max_tokens
46
44 - self.prompt = self.inputs.prompt
45 - self.prompt_token_ids = self.inputs.prompt_token_ids
47 + self.prompt = prompt
48 + self.prompt_token_ids = prompt_token_ids
49 self.num_prompt_tokens = len(self.prompt_token_ids)
50 self._output_token_ids: List[int] = []
51 self._all_token_ids: List[int] = self.prompt_token_ids.copy()
52 self.num_computed_tokens = 0
53
51 - # Multi-modal input metadata.
52 - mm_positions = self.inputs.multi_modal_placeholders
53 - if mm_positions:
54 - # FIXME(woosuk): Support other modalities.
55 - self.mm_positions = mm_positions.get("image", [])
56 - else:
57 - self.mm_positions = []
58 - # Output of the mm input mapper (e.g., image tensors).
59 - self.mm_inputs: List[MultiModalKwargs] = []
60 - if self.inputs.multi_modal_inputs:
61 - self.mm_inputs = self.inputs.multi_modal_inputs
54 + # Multi-modal related
55 + self.mm_positions = multi_modal_placeholders or []
56 + self.mm_inputs = multi_modal_inputs or []
57 + self.mm_hashes: List[str] = multi_modal_hashes or []
58
63 - self.mm_hashes: List[str] = self.inputs.multi_modal_hashes
59 + # Sanity check
60 + assert len(self.mm_inputs) == len(self.mm_positions)
61 + assert len(self.mm_inputs) == len(self.mm_hashes)
62
63 # Cache the computed kv block hashes of the request to avoid
64 # recomputing.
@@ -70,15 +68,11 @@ class Request:
68 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
69 return cls(
70 request_id=request.request_id,
73 - inputs=token_inputs(
74 - prompt_token_ids=request.prompt_token_ids,
75 - prompt=request.prompt,
76 - multi_modal_data=None,
77 - multi_modal_inputs=request.mm_inputs,
78 - multi_modal_hashes=request.mm_hashes,
79 - multi_modal_placeholders=request.mm_placeholders,
80 - mm_processor_kwargs=None,
81 - ),
71 + prompt=request.prompt,
72 + prompt_token_ids=request.prompt_token_ids,
73 + multi_modal_inputs=request.mm_inputs,
74 + multi_modal_hashes=request.mm_hashes,
75 + multi_modal_placeholders=request.mm_placeholders,
76 sampling_params=request.sampling_params,
77 eos_token_id=request.eos_token_id,
78 arrival_time=request.arrival_time,
vllm/v1/worker/gpu_model_runner.py
+38 -36
@@ -19,7 +19,7 @@ from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 LayerBlockType, cdiv, is_pin_memory_available)
20 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 FlashAttentionMetadata)
22 -from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
22 +from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
23 from vllm.v1.outputs import ModelRunnerOutput
24 from vllm.v1.sample.metadata import SamplingMetadata
25 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -82,12 +82,10 @@ class GPUModelRunner:
82 self.input_registry = INPUT_REGISTRY
83 self.mm_registry = MULTIMODAL_REGISTRY
84
85 - # NOTE: mm_input_mapper_client and mm_hasher are only used for memory
86 - # profiling.
87 - self.mm_input_mapper_client = MMInputMapperClient(self.model_config)
88 - self.mm_hasher = MMHasher()
89 - self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
90 - cache_config.enable_prefix_caching
85 + # NOTE: Initialized input mapper is only used for processing dummy
86 + # multimodal data into multimodal kwargs for GPU memory profiling.
87 + self.mm_input_mapper_profiling = MMInputMapperClient(self.model_config)
88 + self.mm_input_mapper_profiling.use_cache = False
89
90 self.max_num_encoder_input_tokens = self.scheduler_config.max_num_encoder_input_tokens # noqa: E501
91 self.encoder_cache_size = self.scheduler_config.encoder_cache_size
@@ -722,8 +720,6 @@ class GPUModelRunner:
720 ]
721
722 # Profile with multimodal encoder & encoder cache.
725 - # TODO (ywang96): generalize this beyond image modality since
726 - # mm_input_mapper only supports image inputs.
723 if self.is_multimodal_model:
724
725 # Create dummy batch of multimodal inputs.
@@ -735,15 +731,30 @@ class GPUModelRunner:
731 dummy_mm_data = dummy_request_data.multi_modal_data
732
733 # NOTE: Currently model is profiled with a single non-text
738 - # modality even when it supports multiple.
739 - max_tokens_per_mm_item = max(
740 - self.mm_registry.get_max_tokens_per_item_by_modality(
741 - self.model_config).values())
742 -
743 - max_num_mm_items_encoder_budget = min(
744 - self.max_num_encoder_input_tokens,
745 - self.encoder_cache_size) // max_tokens_per_mm_item
746 -
734 + # modality with the max possible input tokens even when
735 + # it supports multiple.
736 + max_tokens_by_modality_dict = self.mm_registry.get_max_tokens_per_item_by_modality( # noqa: E501
737 + self.model_config)
738 +
739 + dummy_data_modality, max_tokens_per_mm_item = max(
740 + max_tokens_by_modality_dict.items(), key=lambda item: item[1])
741 +
742 + # Check how many items of this modality can be supported by
743 + # the encoder cache budget.
744 + encoder_cache_budget = min(self.max_num_encoder_input_tokens,
745 + self.encoder_cache_size)
746 + max_num_mm_items_encoder_budget = encoder_cache_budget // \
747 + max_tokens_per_mm_item
748 +
749 + # TODO: Allow users to set encoder_cache_budget in case this
750 + # happens.
751 + assert max_num_mm_items_encoder_budget > 0, (
752 + f"Encoder cache budget={encoder_cache_budget} is too small to "
753 + f"support the maximum possible size of multimodal embeddings"
754 + f"={max_tokens_per_mm_item}.")
755 +
756 + # Check how many items of this modality can be supported by
757 + # the decoder budget.
758 max_mm_items_per_req = max(
759 self.mm_registry.get_mm_limits_per_prompt(
760 self.model_config).values())
@@ -763,33 +774,24 @@ class GPUModelRunner:
774 # they are scheduled to be processed separately.
775
776 # Case when models have a merged processor, their dummy data is
766 - # already batched `MultiModalKwargs`, therefore we need to "unbatch"
767 - # and take the first item in each batched tensor.
768 - # TODO (ywang96): This is somewhat hacky. Refactor this to be
769 - # consistent with the other case.
777 + # already batched `MultiModalKwargs`, therefore we take the first
778 + # `MultiModalKwargsItem` from the desired modality to profile on.
779 if isinstance(dummy_mm_data, MultiModalKwargs):
771 - dummy_mm_kwargs = {
772 - k: v[0].unsqueeze(0)
773 - for k, v in dummy_mm_data.items()
774 - }
780 + dummy_mm_item = dummy_mm_data.get_item(
781 + modality=dummy_data_modality, item_index=0)
782 + dummy_mm_kwargs = MultiModalKwargs.from_items([dummy_mm_item])
783
784 # Case when models have dummy data explicitly defined as
785 # `MultiModalDataDict`, so they need to be processed through input
786 # mapper.
787 + # TODO (ywang96): deprecate this path once merged processor is
788 + # supported on all models.
789 else:
780 - # Compute MM hashes (if enabled)
781 - mm_hashes = None
782 - if self.use_hash:
783 - mm_hashes = self.mm_hasher.hash_dummy_mm_data(
784 - dummy_mm_data)
785 -
786 - mm_kwargs_list = self.mm_input_mapper_client.process_inputs(
790 + mm_kwargs_list = self.mm_input_mapper_profiling.process_inputs(
791 mm_data=dummy_mm_data,
788 - mm_hashes=mm_hashes,
792 + mm_hashes=None,
793 mm_processor_kwargs=None,
794 precomputed_mm_inputs=None)
791 -
792 - # Take the first `MultiModalKwargs`
795 dummy_mm_kwargs = mm_kwargs_list[0]
796
797 batched_dummy_mm_inputs = MultiModalKwargs.batch(