@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c6e7404cc5

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 10:00 UTC

[Multimodal] Simplify MM input definitions (#33331)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 29, 2026 at 21:32 UTC c6e7404cc5713a926e8b6c187b5f197a5436e9ff

17 files changed +142 -164

tests/distributed/test_shm_storage.py

+5 -7

@@ -23,18 +23,16 @@ from vllm.multimodal.inputs import (

23 )

26 -def _dummy_elem(modality: str, key: str, size: int):

26 +def _dummy_elem(size: int):

27 return MultiModalFieldElem(

28 - modality=modality,

29 - key=key,

28 data=torch.empty((size,), dtype=torch.int8),

29 field=MultiModalSharedField(batch_size=1),

30 )

35 -def _dummy_item(modality: str, size_by_key: dict[str, int]):

36 - return MultiModalKwargsItem.from_elems(

37 - [_dummy_elem(modality, key, size) for key, size in size_by_key.items()]

33 +def _dummy_item(size_by_key: dict[str, int]):

34 + return MultiModalKwargsItem(

35 + {key: _dummy_elem(size) for key, size in size_by_key.items()}

36 )

@@ -61,7 +59,7 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase):

59 def test_minimal_put_get_cycle(self):

60 """Test basic put and get operations."""

61 key = "test_key"

64 - value = _dummy_item("text", {"field1": 10, "field2": 20})

62 + value = _dummy_item({"field1": 10, "field2": 20})

64 # Put operation

65 address, monotonic_id = self.storage.put(key, value)

tests/models/multimodal/processing/test_tensor_schema.py

+5 -1

@@ -119,7 +119,11 @@ def create_batched_mm_kwargs(

119 )["mm_kwargs"].require_data()

120

121 return group_mm_kwargs_by_modality(

122 - [item for modality in supported_mm_limits for item in mm_kwargs[modality]]

122 + [

123 + (modality, item)

124 + for modality in supported_mm_limits

125 + for item in mm_kwargs[modality]

126 + ]

127 )

128

129

tests/multimodal/test_cache.py

+15 -21

@@ -36,8 +36,6 @@ pytestmark = pytest.mark.cpu_test

38 def _dummy_elem(

39 - modality: str,

40 - key: str,

39 size: int,

40 *,

41 rng: np.random.RandomState | None = None,

@@ -48,21 +46,18 @@ def _dummy_elem(

46 data = torch.from_numpy(rng.randint(4, size=(size,), dtype=np.int8))

48 return MultiModalFieldElem(

51 - modality=modality,

52 - key=key,

49 data=data,

50 field=MultiModalSharedField(batch_size=1),

51 )

54 def _dummy_item(

59 - modality: str,

55 size_by_key: dict[str, int],

56 *,

57 rng: np.random.RandomState | None = None,

58 ):

64 - return MultiModalKwargsItem.from_elems(

65 - [_dummy_elem(modality, key, size, rng=rng) for key, size in size_by_key.items()]

59 + return MultiModalKwargsItem(

60 + {key: _dummy_elem(size, rng=rng) for key, size in size_by_key.items()}

61 )

@@ -71,19 +66,19 @@ def _dummy_items(

66 *,

67 rng: np.random.RandomState | None = None,

68 ):

74 - return MultiModalKwargsItems.from_seq(

75 - [

76 - _dummy_item(modality, size_by_key, rng=rng)

69 + return MultiModalKwargsItems(

70 + {

71 + modality: [_dummy_item(size_by_key, rng=rng)]

72 for modality, size_by_key in size_by_key_modality.items()

78 - ]

73 + }

74 )

77 @pytest.mark.parametrize(

78 ("item", "expected_size"),

79 [

85 - (_dummy_item("a", {"a1": 100}), 100),

86 - (_dummy_item("a", {"a1": 100, "a2": 110}), 210),

80 + (_dummy_item({"a1": 100}), 100),

81 + (_dummy_item({"a1": 100, "a2": 110}), 210),

82 (_dummy_items({"a": {"a1": 100, "a2": 110}, "b": {"b1": 120, "b2": 130}}), 460), # noqa: E501

83 ],

84 )

@@ -143,7 +138,7 @@ def _compare_caches(

138

139 rng = np.random.RandomState(seed)

140 all_items = [

146 - _dummy_item("item", {"key": item_size_gb}, rng=rng)

141 + _dummy_item({"key": item_size_gb}, rng=rng)

142 for _ in range(int(item_capacity / hit_rate))

143 ]

144 all_hashes = [

@@ -245,13 +240,13 @@ def _run_test_cache_eviction_lru(

240 "image_C",

241 ]

242 request1_items = {

248 - h: MultiModalKwargsItem.dummy(h, nbytes=2 * base_item_size)

243 + h: MultiModalKwargsItem.dummy(nbytes=2 * base_item_size)

244 for h in request1_hashes

245 }

246

247 request2_hashes = ["image_D", "image_E", "image_A", "image_C"]

248 request2_items = {

254 - h: MultiModalKwargsItem.dummy(h, nbytes=1 * base_item_size)

249 + h: MultiModalKwargsItem.dummy(nbytes=1 * base_item_size)

250 for h in request2_hashes

251 }

252

@@ -356,15 +351,14 @@ def _run_test_cache_eviction_shm(

351 ):

352 request1_hashes = ["image_A", "image_B", "image_C"]

353 request1_items = {

359 - h: MultiModalKwargsItem.dummy(h, nbytes=5 * base_item_size)

360 - for h in request1_hashes

354 + h: MultiModalKwargsItem.dummy(5 * base_item_size) for h in request1_hashes

355 }

356 request1_items_p0_result = []

357

358 request2_hashes = ["image_G", "image_A"]

359 request2_items = {

360 h: MultiModalKwargsItem.dummy(

367 - h, nbytes=(5 if h in request1_hashes else 2) * base_item_size

361 + (5 if h in request1_hashes else 2) * base_item_size

362 )

363 for h in request2_hashes

364 }

@@ -373,7 +367,7 @@ def _run_test_cache_eviction_shm(

367 request3_hashes = ["image_G", "image_H", "image_I", "image_B"]

368 request3_items = {

369 h: MultiModalKwargsItem.dummy(

376 - h, nbytes=(5 if h in request1_hashes else 2) * base_item_size

370 + (5 if h in request1_hashes else 2) * base_item_size

371 )

372 for h in request3_hashes

373 }

@@ -532,7 +526,7 @@ def test_processor_cache_shared_across_loras():

526 lora_a_identifier = f"12345:{base_mm_hash}"

527 lora_b_identifier = f"67890:{base_mm_hash}"

528

535 - item_data = MultiModalKwargsItem.dummy("test_image", nbytes=1024)

529 + item_data = MultiModalKwargsItem.dummy(1024)

530

531 feature_lora_a = MultiModalFeatureSpec(

532 data=item_data,

tests/v1/core/test_kv_cache_utils.py

+1 -1

@@ -77,7 +77,7 @@ def make_request(

77 for j, position in enumerate(mm_positions):

78 identifier = mm_hashes[j] if mm_hashes else f"hash_{j}"

79 mm_feature = MultiModalFeatureSpec(

80 - data=MultiModalKwargsItem.dummy("dummy_m"),

80 + data=MultiModalKwargsItem.dummy(),

81 mm_position=position,

82 identifier=identifier,

83 modality="image",

tests/v1/core/test_prefix_caching.py

+1 -1

@@ -68,7 +68,7 @@ def make_request(

68 for j, position in enumerate(mm_positions):

69 identifier = mm_hashes[j] if mm_hashes else f"hash_{j}"

70 mm_feature = MultiModalFeatureSpec(

71 - data=MultiModalKwargsItem.dummy("dummy_m"),

71 + data=MultiModalKwargsItem.dummy(),

72 mm_position=position,

73 identifier=identifier,

74 modality="image",

tests/v1/core/test_priority_scheduler_random.py

+1 -1

@@ -56,7 +56,7 @@ def _create_random_request(

56 for j, position in enumerate(mm_positions):

57 identifier = f"{request_id}_hash_{j}"

58 mm_feature = MultiModalFeatureSpec(

59 - data=MultiModalKwargsItem.dummy("dummy_m"),

59 + data=MultiModalKwargsItem.dummy(),

60 mm_position=position,

61 identifier=identifier,

62 modality="image",

tests/v1/core/test_scheduler.py

+1 -1

@@ -1707,7 +1707,7 @@ def create_requests_with_priority(

1707 # Unique dummy hash for each mm item

1708 identifier = f"hash{i}_{j}"

1709 mm_feature = MultiModalFeatureSpec(

1710 - data=MultiModalKwargsItem.dummy("dummy_m"),

1710 + data=MultiModalKwargsItem.dummy(),

1711 mm_position=position,

1712 identifier=identifier,

1713 modality="image",

tests/v1/core/utils.py

+1 -1

@@ -236,7 +236,7 @@ def create_requests(

236 # Unique dummy hash for each mm item

237 identifier = f"hash{i}_{j}"

238 mm_feature = MultiModalFeatureSpec(

239 - data=MultiModalKwargsItem.dummy("dummy_m"),

239 + data=MultiModalKwargsItem.dummy(),

240 mm_position=position,

241 identifier=identifier,

242 modality="image",

tests/v1/streaming_input/test_gpu_model_runner_streaming.py

+2 -2

@@ -131,7 +131,7 @@ def test_e2e_streaming_with_multimodal_features(mock_model_runner_with_input_bat

131

132 # Step 1: Create initial request state with one multimodal feature

133 mm_feature_1 = MultiModalFeatureSpec(

134 - data=MultiModalKwargsItem.dummy("audio"),

134 + data=MultiModalKwargsItem.dummy(),

135 modality="audio",

136 identifier="audio_1",

137 mm_position=PlaceholderRange(offset=2, length=10),

@@ -158,7 +158,7 @@ def test_e2e_streaming_with_multimodal_features(mock_model_runner_with_input_bat

158 # The scheduler has already set prompt_token_ids to the full sequence

159 # (original prompt + intermediate outputs + new prompt with new multimodal feature)

160 mm_feature_2 = MultiModalFeatureSpec(

161 - data=MultiModalKwargsItem.dummy("audio"),

161 + data=MultiModalKwargsItem.dummy(),

162 modality="audio",

163 identifier="audio_2",

164 mm_position=PlaceholderRange(offset=15, length=5),

tests/v1/streaming_input/test_scheduler_streaming.py

+2 -2

@@ -174,7 +174,7 @@ class TestStreamingScheduler(unittest.TestCase):

174 scheduler = create_scheduler()

175

176 mm_feature = MultiModalFeatureSpec(

177 - data=MultiModalKwargsItem.dummy("audio"),

177 + data=MultiModalKwargsItem.dummy(),

178 modality="audio",

179 identifier="",

180 mm_position=PlaceholderRange(offset=1, length=1),

@@ -187,7 +187,7 @@ class TestStreamingScheduler(unittest.TestCase):

187 session.num_computed_tokens = len(session.prompt_token_ids)

188

189 mm_feature = MultiModalFeatureSpec(

190 - data=MultiModalKwargsItem.dummy("audio"),

190 + data=MultiModalKwargsItem.dummy(),

191 modality="audio",

192 identifier="",

193 mm_position=PlaceholderRange(offset=2, length=1),

tests/v1/test_serial_utils.py

+9 -14

@@ -104,14 +104,10 @@ class MyRequest(msgspec.Struct):

104

105 def test_multimodal_kwargs():

106 e1 = MultiModalFieldElem(

107 - "audio",

108 - "a0",

107 torch.zeros(1000, dtype=torch.bfloat16),

108 MultiModalBatchedField(),

109 )

110 e2 = MultiModalFieldElem(

113 - "video",

114 - "v0",

111 [torch.zeros(1000, dtype=torch.int8) for _ in range(4)],

112 MultiModalFlatField(

113 slices=[[slice(1, 2, 3), slice(4, 5, 6)], [slice(None, 2)]],

@@ -119,21 +115,20 @@ def test_multimodal_kwargs():

115 ),

116 )

117 e3 = MultiModalFieldElem(

122 - "image",

123 - "i0",

118 torch.zeros(1000, dtype=torch.int32),

119 MultiModalSharedField(batch_size=4),

120 )

121 e4 = MultiModalFieldElem(

128 - "image",

129 - "i1",

122 torch.zeros(1000, dtype=torch.int32),

123 MultiModalFlatField(slices=[slice(1, 2, 3), slice(4, 5, 6)], dim=2),

124 )

133 - audio = MultiModalKwargsItem.from_elems([e1])

134 - video = MultiModalKwargsItem.from_elems([e2])

135 - image = MultiModalKwargsItem.from_elems([e3, e4])

136 - mm = MultiModalKwargsItems.from_seq([audio, video, image])

125 + mm = MultiModalKwargsItems(

126 + {

127 + "audio": [MultiModalKwargsItem({"a0": e1})],

128 + "video": [MultiModalKwargsItem({"v0": e2})],

129 + "image": [MultiModalKwargsItem({"i0": e3, "i1": e4})],

130 + }

131 + )

132

133 # pack mm kwargs into a mock request so that it can be decoded properly

134 req = MyRequest([mm])

@@ -147,8 +142,8 @@ def test_multimodal_kwargs():

142

143 total_len = sum(memoryview(x).cast("B").nbytes for x in encoded)

144

150 - # expected total encoding length, should be 14395, +-20 for minor changes

151 - assert 14375 <= total_len <= 14425

145 + # expected total encoding length, should be 14319, +-20 for minor changes

146 + assert 14300 <= total_len <= 14340

147 decoded = decoder.decode(encoded).mm[0]

148 assert isinstance(decoded, MultiModalKwargsItems)

149

vllm/multimodal/cache.py

+14 -17

@@ -463,8 +463,8 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

463 ring_buffer=ring_buffer,

464 serde_class=MsgpackSerde,

465 )

466 - # cache (prompt_updates, modality) for P0 only

467 - self._p0_cache: dict[str, tuple[Sequence[ResolvedPromptUpdate], str]] = {}

466 + # cache prompt_updates for P0 only

467 + self._p0_cache: dict[str, Sequence[ResolvedPromptUpdate]] = {}

468

469 self._hits = 0

470 self._total = 0

@@ -495,23 +495,22 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

495 self._total += 1

496

497 address, monotonic_id = self._shm_cache.get_cached(mm_hash)

498 - prompt_updates, modality = self._p0_cache[mm_hash]

499 - return self.address_as_item(address, monotonic_id, modality), prompt_updates

498 + prompt_updates = self._p0_cache[mm_hash]

499 + return self.address_as_item(address, monotonic_id), prompt_updates

500

501 assert mm_item is not None, f"Expected a cached item for {mm_hash=}"

502 + item, prompt_updates = mm_item

503

504 self._total += 1

505

506 try:

506 - address, monotonic_id = self._shm_cache.put(mm_hash, mm_item[0])

507 + address, monotonic_id = self._shm_cache.put(mm_hash, item)

508 # Try to remove dangling items if p0 cache is too large.

509 if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index):

510 self.remove_dangling_items()

510 - self._p0_cache[mm_hash] = mm_item[1], mm_item[0].modality

511 - address_item = self.address_as_item(

512 - address, monotonic_id, mm_item[0].modality

513 - )

514 - return address_item, mm_item[1]

511 +

512 + self._p0_cache[mm_hash] = prompt_updates

513 + return self.address_as_item(address, monotonic_id), prompt_updates

514 except (ValueError, MemoryError) as e:

515 # put may fail if the object is too large or

516 # the cache is full.

@@ -550,22 +549,20 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache):

549 del self._p0_cache[mm_hash]

550

551 def address_as_item(

553 - self, address: int, monotonic_id: int, modality: str

552 + self,

553 + address: int,

554 + monotonic_id: int,

555 ) -> MultiModalKwargsItem:

556 addr_elem = MultiModalFieldElem(

556 - modality=modality,

557 - key="address",

557 data=address,

558 field=MultiModalBatchedField(),

559 )

560 id_elem = MultiModalFieldElem(

562 - modality=modality,

563 - key="monotonic_id",

561 data=monotonic_id,

562 field=MultiModalBatchedField(),

563 )

567 - mm_item = MultiModalKwargsItem.from_elems([addr_elem, id_elem])

568 - return mm_item

564 +

565 + return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem})

566

567

568 class BaseMultiModalReceiverCache(

vllm/multimodal/inputs.py

+65 -72

@@ -23,7 +23,7 @@ import numpy as np

23 from PIL.Image import Image

24 from typing_extensions import NotRequired, TypeVar

26 -from vllm.utils.collection_utils import full_groupby, is_list_of

26 +from vllm.utils.collection_utils import is_list_of

27 from vllm.utils.import_utils import LazyLoader

28 from vllm.utils.jsontree import json_map_leaves

@@ -336,25 +336,33 @@ class MultiModalFeatureSpec:

336 """

337 Represents a single multimodal input with its processed data and metadata.

338

339 - Used by the V1 engine to track multimodal data through processing and

340 - caching. A request containing multiple multimodal items will have one

341 - MultiModalFeatureSpec per item.

339 + Used to track multimodal data through processing and caching.

340 + A request containing multiple multimodal items will have one

341 + `MultiModalFeatureSpec` per item.

342 """

343

344 data: Optional["MultiModalKwargsItem"]

345 - """Multimodal data for this feature"""

345 + """

346 + Represents multimodal data for this feature.

347 +

348 + Can be `None` if the item is cached, to skip IPC between API server

349 + and engine core processes.

350 + """

351

352 modality: str

348 - """Based on the input, e.g., "image", "audio", "video"."""

353 + """The input modality, e.g., `"image"`, `"audio"`, `"video"`."""

354

355 identifier: str

351 - """mm_hash or uuid for caching encoder outputs."""

356 + """The hash for caching encoder outputs (with LoRA prefix if applicable)."""

357

358 mm_position: PlaceholderRange

354 - """e.g., PlaceholderRange(offset=2, length=336)"""

359 + """

360 + The location of the `modality` tokens corresponding to this item

361 + in the prompt, e.g., `PlaceholderRange(offset=2, length=336)`.

362 + """

363

364 mm_hash: str | None = None

357 - """Base mm_hash for processor cache (without LoRA prefix)."""

365 + """The hash for caching processor outputs (without LoRA prefix)."""

366

367 @staticmethod

368 def gather_kwargs(features: list["MultiModalFeatureSpec"], keys: set[str]):

@@ -373,23 +381,10 @@ class MultiModalFeatureSpec:

381 @dataclass

382 class MultiModalFieldElem:

383 """

376 - Represents a keyword argument inside a

384 + Represents a processed keyword argument to pass to a model for a

385 [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem].

386 """

387

380 - modality: str

381 - """

382 - The modality of the corresponding multi-modal item.

383 - Each multi-modal item can consist of multiple keyword arguments.

384 - """

385 -

386 - key: str

387 - """

388 - The key of this field in

389 - [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem],

390 - i.e. the name of the keyword argument to be passed to the model.

391 - """

392 -

388 data: NestedTensors

389 """

390 The tensor data of this field in

@@ -417,11 +412,7 @@ class MultiModalFieldElem:

412 else:

413 data_equal = nested_tensors_equal(self.data, other.data)

414

420 - return (

421 - (self.modality, self.key) == (other.modality, other.key)

422 - and data_equal

423 - and type(self.field) is type(other.field)

424 - ) # noqa: E721

415 + return data_equal and type(self.field) is type(other.field) # noqa: E721

416

417

418 @dataclass(frozen=True, kw_only=True)

@@ -438,13 +429,8 @@ class BaseMultiModalField(ABC):

429 when `MultiModalKwargsItems.get_data()` is called to batch the data.

430 """

431

441 - def _field_factory(self, *, modality: str, key: str):

442 - f = partial(

443 - MultiModalFieldElem,

444 - modality=modality,

445 - key=key,

446 - field=self,

447 - )

432 + def _field_factory(self):

433 + f = partial(MultiModalFieldElem, field=self)

434

435 # Allow passing data as positional argument

436 def factory(data: NestedTensors) -> MultiModalFieldElem:

@@ -519,7 +505,7 @@ class MultiModalBatchedField(BaseMultiModalField):

505 key: str,

506 data: NestedTensors,

507 ) -> Sequence[MultiModalFieldElem]:

522 - field_factory = self._field_factory(modality=modality, key=key)

508 + field_factory = self._field_factory()

509 return [field_factory(item) for item in data]

510

511 def _reduce_data(

@@ -565,7 +551,7 @@ class MultiModalFlatField(BaseMultiModalField):

551 key: str,

552 data: NestedTensors,

553 ) -> Sequence[MultiModalFieldElem]:

568 - field_factory = self._field_factory(modality=modality, key=key)

554 + field_factory = self._field_factory()

555 if not is_list_of(self.slices, slice, check="all"):

556 assert isinstance(data, torch.Tensor), (

557 "torch.Tensor is required for multiple slices"

@@ -664,7 +650,7 @@ class MultiModalSharedField(BaseMultiModalField):

650 key: str,

651 data: NestedTensors,

652 ) -> Sequence[MultiModalFieldElem]:

667 - field_factory = self._field_factory(modality=modality, key=key)

653 + field_factory = self._field_factory()

654 return [field_factory(data)] * self.batch_size

655

656 def _reduce_data(

@@ -899,37 +885,19 @@ class MultiModalFieldConfig:

885

886 class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):

887 """

902 - A collection of

903 - [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]

904 - corresponding to a data item in

888 + A dictionary of processed keyword arguments to pass to the model,

889 + corresponding to a single item in

890 [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].

891 """

892

893 @staticmethod

909 - def dummy(modality: str, nbytes: int = 1):

894 + def dummy(nbytes: int = 1):

895 """Convenience class for testing."""

896 mm_elem = MultiModalFieldElem(

912 - modality=modality,

913 - key="dummy",

897 data=torch.empty(nbytes, dtype=torch.uint8),

898 field=MultiModalSharedField(batch_size=1),

899 )

917 - return MultiModalKwargsItem.from_elems([mm_elem])

918 -

919 - @staticmethod

920 - def from_elems(elems: Sequence[MultiModalFieldElem]):

921 - return MultiModalKwargsItem({elem.key: elem for elem in elems})

922 -

923 - def __init__(self, data: Mapping[str, MultiModalFieldElem] = {}) -> None:

924 - super().__init__(data)

925 -

926 - modalities = {elem.modality for elem in self.values()}

927 - assert len(modalities) == 1, f"Found different modalities={modalities}"

928 - self._modality = next(iter(modalities))

929 -

930 - @property

931 - def modality(self) -> str:

932 - return self._modality

900 + return MultiModalKwargsItem({"dummy": mm_elem})

901

902 def get_data(self) -> dict[str, NestedTensors]:

903 return {key: elem.data for key, elem in self.items()}

@@ -945,9 +913,38 @@ _I = TypeVar(

913

914 class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):

915 """

948 - A dictionary of

949 - [`MultiModalKwargsItem`][vllm.multimodal.inputs.MultiModalKwargsItem]s

950 - by modality.

916 + A dictionary of processed multi-modal inputs by modality.

917 +

918 + For example, given a processor that processes

919 + images into `pixel_values` and `image_grid_thw`,

920 + and audios into `input_audio_features`,

921 + a prompt with 2 images and 1 audio will be processed

922 + into a `MultiModalKwargsItems` with the following structure:

923 +

924 + ```python

925 + MultiModalKwargsItems(

926 + {

927 + "image": [

928 + # For the first image

929 + MultiModalKwargsItem({"pixel_values": ..., "image_grid_thw": ...}),

930 + # For the second imgae

931 + MultiModalKwargsItem({"pixel_values": ..., "image_grid_thw": ...}),

932 + ],

933 + "audio": [

934 + # For the first audio

935 + MultiModalKwargsItem({"input_audio_features": ...}),

936 + ],

937 + }

938 + )

939 + ```

940 +

941 + Unlike HF processing which returns all items

942 + in a single dictionary with batched keyword arguments,

943 + we split up the items because some of them may already be cached.

944 + Also, items from multiple requests may be batched together to improve throughput,

945 + using the logic defined by the

946 + [`BaseMultiModalField`][vllm.multimodal.inputs.BaseMultiModalField]

947 + for each keyword argument.

948 """

949

950 @staticmethod

@@ -967,7 +964,7 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):

964 elems_by_key[key] = elems

965 keys_by_modality[config.modality].add(key)

966

970 - items = list[MultiModalKwargsItem]()

967 + items_by_modality = dict[str, list[MultiModalKwargsItem]]()

968 for modality, keys in keys_by_modality.items():

969 elems_in_modality = {k: elems_by_key[k] for k in keys}

970 batch_sizes = {k: len(v) for k, v in elems_in_modality.items()}

@@ -979,15 +976,11 @@ class MultiModalKwargsItems(UserDict[str, Sequence[_I]]):

976 )

977

978 batch_size = next(iter(batch_sizes.values()))

982 - for item_idx in range(batch_size):

983 - elems = [v[item_idx] for v in elems_in_modality.values()]

984 - items.append(MultiModalKwargsItem.from_elems(elems))

985 -

986 - return MultiModalKwargsItems.from_seq(items)

979 + items_by_modality[modality] = [

980 + MultiModalKwargsItem({k: v[i] for k, v in elems_in_modality.items()})

981 + for i in range(batch_size)

982 + ]

983

988 - @staticmethod

989 - def from_seq(items: Sequence[MultiModalKwargsItem]):

990 - items_by_modality = full_groupby(items, key=lambda x: x.modality)

984 return MultiModalKwargsItems(items_by_modality)

985

986 def __getitem__(self, modality: str) -> Sequence[_I]:

vllm/multimodal/utils.py

+4 -4

@@ -467,7 +467,7 @@ def argsort_mm_positions(

467

468

469 def group_mm_kwargs_by_modality(

470 - mm_kwargs: list[MultiModalKwargsItem],

470 + mm_kwargs: list[tuple[str, MultiModalKwargsItem]],

471 *,

472 device: torch.types.Device = None,

473 pin_memory: bool = False,

@@ -485,9 +485,9 @@ def group_mm_kwargs_by_modality(

485 """

486 from vllm.multimodal.inputs import MultiModalKwargsItems

487

488 - for modality, items in groupby(mm_kwargs, key=lambda item: item.modality):

489 - items_lst = list(items)

490 - mm_kwargs_items = MultiModalKwargsItems.from_seq(items_lst)

488 + for modality, group in groupby(mm_kwargs, key=lambda x: x[0]):

489 + items_lst = [item for _, item in group]

490 + mm_kwargs_items = MultiModalKwargsItems({modality: items_lst})

491 mm_kwargs_data = mm_kwargs_items.get_data(

492 device=device,

493 pin_memory=pin_memory,

vllm/v1/serial_utils.py

+5 -7

@@ -242,13 +242,11 @@ class MsgpackEncoder:

242 for modality, itemlist in items.items()

243 }

244

245 - def _encode_mm_item(self, item: MultiModalKwargsItem) -> list[dict[str, Any]]:

246 - return [self._encode_mm_field_elem(elem) for elem in item.values()]

245 + def _encode_mm_item(self, item: MultiModalKwargsItem) -> dict[str, Any]:

246 + return {key: self._encode_mm_field_elem(elem) for key, elem in item.items()}

247

248 def _encode_mm_field_elem(self, elem: MultiModalFieldElem) -> dict[str, Any]:

249 return {

250 - "modality": elem.modality,

251 - "key": elem.key,

250 "data": (

251 None if elem.data is None else self._encode_nested_tensors(elem.data)

252 ),

@@ -383,9 +381,9 @@ class MsgpackDecoder:

381 }

382 )

383

386 - def _decode_mm_item(self, obj: list[Any]) -> MultiModalKwargsItem:

387 - return MultiModalKwargsItem.from_elems(

388 - [self._decode_mm_field_elem(v) for v in obj]

384 + def _decode_mm_item(self, obj: dict[str, Any]) -> MultiModalKwargsItem:

385 + return MultiModalKwargsItem(

386 + {key: self._decode_mm_field_elem(elem) for key, elem in obj.items()}

387 )

388

389 def _decode_mm_field_elem(self, obj: dict[str, Any]) -> MultiModalFieldElem:

vllm/v1/worker/gpu/mm/encoder_runner.py

+5 -4

@@ -43,9 +43,9 @@ class EncoderRunner:

43 def prepare_mm_inputs(

44 self,

45 scheduled_encoder_inputs: dict[str, list[int]],

46 - ) -> tuple[list[str], list[MultiModalKwargsItem]]:

46 + ) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]:

47 mm_hashes: list[str] = []

48 - mm_kwargs: list[MultiModalKwargsItem] = []

48 + mm_kwargs: list[tuple[str, MultiModalKwargsItem]] = []

49 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():

50 mm_features = self.req_id_to_mm_features[req_id]

51 for mm_input_id in encoder_input_ids:

@@ -53,7 +53,8 @@ class EncoderRunner:

53 if mm_feature.data is None:

54 continue

55 mm_hashes.append(mm_feature.identifier)

56 - mm_kwargs.append(mm_feature.data)

56 + mm_kwargs.append((mm_feature.modality, mm_feature.data))

57 +

58 return mm_hashes, mm_kwargs

60 @torch.inference_mode()

@@ -61,7 +62,7 @@ class EncoderRunner:

62 self,

63 model: SupportsMultiModal,

64 mm_hashes: list[str],

64 - mm_kwargs: list[MultiModalKwargsItem],

65 + mm_kwargs: list[tuple[str, MultiModalKwargsItem]],

66 ) -> list[torch.Tensor]:

67 if not mm_hashes:

68 return []

vllm/v1/worker/gpu_model_runner.py

+6 -8

@@ -1217,11 +1217,11 @@ class GPUModelRunner(

1217 if not scheduler_output or not self.is_multimodal_raw_input_only_model:

1218 return {}

1219

1220 - mm_kwargs = list[MultiModalKwargsItem]()

1220 + mm_kwargs = list[tuple[str, MultiModalKwargsItem]]()

1221 for req in scheduler_output.scheduled_new_reqs:

1222 for feature in req.mm_features:

1223 if feature.data is not None:

1224 - mm_kwargs.append(feature.data)

1224 + mm_kwargs.append((feature.modality, feature.data))

1225

1226 # Input all modalities at once

1227 mm_kwargs_combined: BatchedTensorInputs = {}

@@ -2219,7 +2219,7 @@ class GPUModelRunner(

2219 scheduler_output: "SchedulerOutput",

2220 ) -> tuple[

2221 list[str],

2222 - list[MultiModalKwargsItem],

2222 + list[tuple[str, MultiModalKwargsItem]],

2223 list[tuple[str, PlaceholderRange]],

2224 ]:

2225 """Batch multimodal inputs from scheduled encoder inputs.

@@ -2239,7 +2239,7 @@ class GPUModelRunner(

2239 return [], [], []

2240

2241 mm_hashes = list[str]()

2242 - mm_kwargs = list[MultiModalKwargsItem]()

2242 + mm_kwargs = list[tuple[str, MultiModalKwargsItem]]()

2243 # Multimodal LoRA reference info to map each multimodal item

2244 # back to its request & position

2245 mm_lora_refs = list[tuple[str, PlaceholderRange]]()

@@ -2252,7 +2252,7 @@ class GPUModelRunner(

2252 continue

2253

2254 mm_hashes.append(mm_feature.identifier)

2255 - mm_kwargs.append(mm_feature.data)

2255 + mm_kwargs.append((mm_feature.modality, mm_feature.data))

2256 mm_lora_refs.append((req_id, mm_feature.mm_position))

2257

2258 return mm_hashes, mm_kwargs, mm_lora_refs

@@ -4475,12 +4475,10 @@ class GPUModelRunner(

4475 # but not read from the cache

4476 assert dummy_mm_item is not None, "Item should not already be cached"

4477

4478 - dummy_mm_items = [dummy_mm_item] * max_items_per_batch

4479 -

4478 return next(

4479 mm_kwargs_group

4480 for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(

4483 - dummy_mm_items,

4481 + [(modality, dummy_mm_item)] * max_items_per_batch,

4482 device=self.device,

4483 pin_memory=self.pin_memory,

4484 )