@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 19b927e52d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Core] Use individual MM items in P0/P1 cache and model runner (#22570)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Aug 13, 2025 at 22:18 UTC 19b927e52df8400084df1c8116af7d6f0a5f5d15
24 files changed +548 -485
tests/multimodal/test_utils.py
+79 -154
@@ -5,7 +5,7 @@ import base64
5 import mimetypes
6 import os
7 from tempfile import NamedTemporaryFile, TemporaryDirectory
8 -from typing import TYPE_CHECKING, NamedTuple, Optional
8 +from typing import TYPE_CHECKING, NamedTuple
9
10 import numpy as np
11 import pytest
@@ -19,14 +19,12 @@ from vllm.distributed.parallel_state import (init_distributed_environment,
19 initialize_model_parallel)
20 from vllm.multimodal.image import convert_image_mode
21 from vllm.multimodal.inputs import PlaceholderRange
22 -from vllm.multimodal.utils import (MediaConnector,
23 - merge_and_sort_multimodal_metadata,
22 +from vllm.multimodal.utils import (MediaConnector, argsort_mm_positions,
23 run_dp_sharded_vision_model)
24 from vllm.platforms import current_platform
25 from vllm.utils import get_open_port, update_environment_variables
26
27 if TYPE_CHECKING:
29 - from vllm.multimodal.hasher import MultiModalHashDict
28 from vllm.multimodal.inputs import MultiModalPlaceholderDict
29
30 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
@@ -178,19 +176,17 @@ async def test_fetch_video_http(video_url: str, num_frames: int):
176 assert metadata_sync == metadata_async
177
178
181 -# Used for the next two tests related to `merge_and_sort_multimodal_metadata`.
179 +# Used for `test_argsort_mm_positions`.
180 class TestCase(NamedTuple):
181 mm_positions: "MultiModalPlaceholderDict"
184 - mm_hashes: Optional["MultiModalHashDict"]
185 - expected_modalities: list[str]
186 - expected_ranges: list[PlaceholderRange]
187 - expected_hashes: Optional[list[str]]
182 + expected_modality_idxs: list[tuple[str, int]]
183
184
190 -def test_merge_and_sort_multimodal_metadata():
185 +def test_argsort_mm_positions():
186
187 test_cases = [
193 - # Single modality should return result as is but flattened
188 + # Single modality
189 + ## Internally sorted
190 TestCase(
191 mm_positions={
192 "image": [
@@ -198,34 +194,27 @@ def test_merge_and_sort_multimodal_metadata():
194 PlaceholderRange(offset=3, length=2),
195 ]
196 },
201 - mm_hashes={"image": ["hash1", "hash2"]},
202 - expected_modalities=["image", "image"],
203 - expected_ranges=[
204 - PlaceholderRange(offset=0, length=2),
205 - PlaceholderRange(offset=3, length=2),
197 + expected_modality_idxs=[
198 + ("image", 0),
199 + ("image", 1),
200 ],
207 - expected_hashes=["hash1", "hash2"],
201 ),
209 -
210 - # Single modality without hashes return None for mm hash.
202 + ## Internally unsorted
203 TestCase(
204 mm_positions={
205 "image": [
206 + PlaceholderRange(offset=3, length=2),
207 PlaceholderRange(offset=0, length=2),
215 - PlaceholderRange(offset=2, length=2),
208 ]
209 },
218 - mm_hashes=None,
219 - expected_modalities=["image", "image"],
220 - expected_ranges=[
221 - PlaceholderRange(offset=0, length=2),
222 - PlaceholderRange(offset=2, length=2),
210 + expected_modality_idxs=[
211 + ("image", 1),
212 + ("image", 0),
213 ],
224 - expected_hashes=None,
214 ),
215
227 - # Multiple modalities with hashes should return sorted modalities
228 - # and flattened ranges and hashes.
216 + # Two modalities
217 + ## Internally sorted
218 TestCase(
219 mm_positions={
220 "image": [
@@ -237,47 +226,54 @@ def test_merge_and_sort_multimodal_metadata():
226 PlaceholderRange(offset=2, length=3),
227 ]
228 },
240 - mm_hashes={
241 - "image": ["image_hash1", "image_hash2"],
242 - "audio": ["audio_hash1", "audio_hash2"],
243 - },
244 - expected_modalities=["audio", "audio", "image", "image"],
245 - expected_ranges=[
246 - PlaceholderRange(offset=0, length=2),
247 - PlaceholderRange(offset=2, length=3),
248 - PlaceholderRange(offset=7, length=4),
249 - PlaceholderRange(offset=11, length=5),
229 + expected_modality_idxs=[
230 + ("audio", 0),
231 + ("audio", 1),
232 + ("image", 0),
233 + ("image", 1),
234 ],
251 - expected_hashes=[
252 - "audio_hash1", "audio_hash2", "image_hash1", "image_hash2"
235 + ),
236 + ## Interleaved, internally sorted
237 + TestCase(
238 + mm_positions={
239 + "image": [
240 + PlaceholderRange(offset=0, length=4),
241 + PlaceholderRange(offset=8, length=2),
242 + ],
243 + "audio": [
244 + PlaceholderRange(offset=5, length=2),
245 + PlaceholderRange(offset=11, length=4),
246 + ]
247 + },
248 + expected_modality_idxs=[
249 + ("image", 0),
250 + ("audio", 0),
251 + ("image", 1),
252 + ("audio", 1),
253 ],
254 ),
255 -
256 - # Multiple modalities without hashes should return sorted modalities
257 - # and flattened ranges and None.
255 + ## Interleaved, internally unsorted
256 TestCase(
257 mm_positions={
258 "image": [
261 - PlaceholderRange(offset=7, length=4),
262 - PlaceholderRange(offset=11, length=5),
259 + PlaceholderRange(offset=8, length=2),
260 + PlaceholderRange(offset=0, length=4),
261 ],
262 "audio": [
265 - PlaceholderRange(offset=0, length=2),
266 - PlaceholderRange(offset=2, length=3),
263 + PlaceholderRange(offset=11, length=4),
264 + PlaceholderRange(offset=5, length=2),
265 ]
266 },
269 - mm_hashes=None,
270 - expected_modalities=["audio", "audio", "image", "image"],
271 - expected_ranges=[
272 - PlaceholderRange(offset=0, length=2),
273 - PlaceholderRange(offset=2, length=3),
274 - PlaceholderRange(offset=7, length=4),
275 - PlaceholderRange(offset=11, length=5),
267 + expected_modality_idxs=[
268 + ("image", 1),
269 + ("audio", 1),
270 + ("image", 0),
271 + ("audio", 0),
272 ],
277 - expected_hashes=None,
273 ),
274
275 # Three modalities
276 + ## Internally sorted
277 TestCase(
278 mm_positions={
279 "image": [
@@ -293,72 +289,16 @@ def test_merge_and_sort_multimodal_metadata():
289 PlaceholderRange(offset=12, length=6),
290 ]
291 },
296 - mm_hashes={
297 - "image": ["image_hash1", "image_hash2"],
298 - "audio": ["audio_hash1"],
299 - "video": ["video_hash1", "video_hash2", "video_hash3"]
300 - },
301 - expected_modalities=[
302 - "audio", "video", "video", "video", "image", "image"
303 - ],
304 - expected_ranges=[
305 - PlaceholderRange(offset=0, length=2),
306 - PlaceholderRange(offset=3, length=4),
307 - PlaceholderRange(offset=7, length=5),
308 - PlaceholderRange(offset=12, length=6),
309 - PlaceholderRange(offset=15, length=7),
310 - PlaceholderRange(offset=22, length=8),
311 - ],
312 - expected_hashes=[
313 - "audio_hash1", "video_hash1", "video_hash2", "video_hash3",
314 - "image_hash1", "image_hash2"
315 - ],
316 - ),
317 - ]
318 -
319 - for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
320 - expected_hashes) in test_cases:
321 - modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
322 - mm_positions, mm_hashes)
323 -
324 - assert modalities == expected_modalities
325 - assert ranges == expected_ranges
326 - assert hashes == expected_hashes
327 -
328 -
329 -def test_merge_and_sort_multimodal_metadata_with_interleaving():
330 -
331 - test_cases = [
332 -
333 - # <image> <audio> <image> <audio>
334 - TestCase(
335 - mm_positions={
336 - "image": [
337 - PlaceholderRange(offset=0, length=4),
338 - PlaceholderRange(offset=8, length=2),
339 - ],
340 - "audio": [
341 - PlaceholderRange(offset=5, length=2),
342 - PlaceholderRange(offset=11, length=4),
343 - ]
344 - },
345 - mm_hashes={
346 - "image": ["image_hash1", "image_hash2"],
347 - "audio": ["audio_hash1", "audio_hash2"],
348 - },
349 - expected_modalities=["image", "audio", "image", "audio"],
350 - expected_ranges=[
351 - PlaceholderRange(offset=0, length=4),
352 - PlaceholderRange(offset=5, length=2),
353 - PlaceholderRange(offset=8, length=2),
354 - PlaceholderRange(offset=11, length=4),
355 - ],
356 - expected_hashes=[
357 - "image_hash1", "audio_hash1", "image_hash2", "audio_hash2"
292 + expected_modality_idxs=[
293 + ("audio", 0),
294 + ("video", 0),
295 + ("video", 1),
296 + ("video", 2),
297 + ("image", 0),
298 + ("image", 1),
299 ],
300 ),
360 -
361 - # <image> <image> <audio> <video> <image>
301 + ## Interleaved, internally sorted
302 TestCase(
303 mm_positions={
304 "image": [
@@ -373,58 +313,43 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
313 PlaceholderRange(offset=8, length=5),
314 ]
315 },
376 - mm_hashes=None,
377 - expected_modalities=["image", "image", "audio", "video", "image"],
378 - expected_ranges=[
379 - PlaceholderRange(offset=0, length=2),
380 - PlaceholderRange(offset=2, length=3),
381 - PlaceholderRange(offset=5, length=2),
382 - PlaceholderRange(offset=8, length=5),
383 - PlaceholderRange(offset=20, length=4),
316 + expected_modality_idxs=[
317 + ("image", 0),
318 + ("image", 1),
319 + ("audio", 0),
320 + ("video", 0),
321 + ("image", 2),
322 ],
385 - expected_hashes=None,
323 ),
387 -
388 - # <image> <audio> <video> <image> with hashes
324 + ## Interleaved, internally sunorted
325 TestCase(
326 mm_positions={
327 "image": [
328 PlaceholderRange(offset=0, length=2),
393 - PlaceholderRange(offset=18, length=4),
329 + PlaceholderRange(offset=20, length=4),
330 + PlaceholderRange(offset=2, length=3),
331 ],
332 "audio": [
396 - PlaceholderRange(offset=6, length=2),
333 + PlaceholderRange(offset=5, length=2),
334 ],
335 "video": [
399 - PlaceholderRange(offset=10, length=5),
336 + PlaceholderRange(offset=8, length=5),
337 ]
338 },
402 - mm_hashes={
403 - "image": ["image_hash1", "image_hash2"],
404 - "audio": ["audio_hash1"],
405 - "video": ["video_hash1"],
406 - },
407 - expected_modalities=["image", "audio", "video", "image"],
408 - expected_ranges=[
409 - PlaceholderRange(offset=0, length=2),
410 - PlaceholderRange(offset=6, length=2),
411 - PlaceholderRange(offset=10, length=5),
412 - PlaceholderRange(offset=18, length=4),
413 - ],
414 - expected_hashes=[
415 - "image_hash1", "audio_hash1", "video_hash1", "image_hash2"
339 + expected_modality_idxs=[
340 + ("image", 0),
341 + ("image", 2),
342 + ("audio", 0),
343 + ("video", 0),
344 + ("image", 1),
345 ],
346 ),
347 ]
348
420 - for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
421 - expected_hashes) in test_cases:
422 - modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
423 - mm_positions, mm_hashes)
349 + for mm_positions, expected_modality_idxs in test_cases:
350 + modality_idxs = argsort_mm_positions(mm_positions)
351
425 - assert modalities == expected_modalities
426 - assert ranges == expected_ranges
427 - assert hashes == expected_hashes
352 + assert modality_idxs == expected_modality_idxs
353
354
355 class SimpleLinearModel(torch.nn.Module):
tests/v1/core/test_kv_cache_utils.py
+30 -18
@@ -1,12 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import importlib
4 +from typing import Optional
5
6 import pytest
7 import torch
8
9 from vllm.config import ModelConfig, SchedulerConfig, VllmConfig
9 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
10 +from vllm.multimodal.inputs import (MultiModalBatchedField,
11 + MultiModalFieldElem, MultiModalKwargsItem,
12 + PlaceholderRange)
13 from vllm.sampling_params import SamplingParams
14 from vllm.utils import GiB_bytes, sha256, sha256_cbor_64bit
15 from vllm.v1.core.kv_cache_manager import KVCacheManager
@@ -27,20 +30,29 @@ from vllm.v1.request import Request
30 # yapf: enable
31
32
30 -def make_request(request_id,
31 - prompt_token_ids,
32 - mm_positions=None,
33 - mm_hashes=None,
34 - cache_salt=None):
33 +def make_request(
34 + request_id: str,
35 + prompt_token_ids: list[int],
36 + mm_positions: Optional[list[PlaceholderRange]] = None,
37 + mm_hashes: Optional[list[str]] = None,
38 + cache_salt: Optional[str] = None,
39 +):
40 if mm_positions is None:
36 - multi_modal_inputs = None
41 + mm_kwargs = None
42 else:
38 - multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
43 + mm_elem = MultiModalFieldElem(
44 + modality="dummy_m",
45 + key="dummy_k",
46 + data=None,
47 + field=MultiModalBatchedField(),
48 + )
49 + mm_item = MultiModalKwargsItem.from_elems([mm_elem])
50 + mm_kwargs = [mm_item] * len(mm_positions)
51
52 return Request(
53 request_id=request_id,
54 prompt_token_ids=prompt_token_ids,
43 - multi_modal_inputs=multi_modal_inputs,
55 + multi_modal_kwargs=mm_kwargs,
56 multi_modal_hashes=mm_hashes,
57 multi_modal_placeholders=mm_positions,
58 sampling_params=SamplingParams(max_tokens=17),
@@ -316,7 +328,7 @@ def test_free_kv_cache_block_queue_get_all_free_blocks():
328
329 def test_generate_block_hash_extra_keys():
330 request = make_request(
319 - request_id=0,
331 + request_id="0",
332 prompt_token_ids=[_ for _ in range(20)],
333 mm_positions=[
334 PlaceholderRange(offset=0, length=5),
@@ -348,7 +360,7 @@ def test_generate_block_hash_extra_keys():
360
361 def test_generate_block_hash_extra_keys_no_mm_inputs():
362 request = make_request(
351 - request_id=0,
363 + request_id="0",
364 prompt_token_ids=[_ for _ in range(6)],
365 mm_positions=None,
366 mm_hashes=None,
@@ -361,7 +373,7 @@ def test_generate_block_hash_extra_keys_no_mm_inputs():
373
374 def test_generate_block_hash_extra_keys_cache_salt():
375 request = make_request(
364 - request_id=0,
376 + request_id="0",
377 prompt_token_ids=[_ for _ in range(6)],
378 mm_positions=None,
379 mm_hashes=None,
@@ -382,7 +394,7 @@ def test_generate_block_hash_extra_keys_cache_salt():
394
395 # works together with other extra keys
396 request_mm = make_request(
385 - request_id=0,
397 + request_id="0",
398 prompt_token_ids=[_ for _ in range(20)],
399 mm_positions=[
400 PlaceholderRange(offset=0, length=5),
@@ -420,7 +432,7 @@ def test_hash_request_tokens(hash_fn):
432 import vllm.v1.core.kv_cache_utils
433 init_none_hash(hash_fn)
434 request = make_request(
423 - request_id=0,
435 + request_id="0",
436 prompt_token_ids=[_ for _ in range(6)],
437 mm_positions=[
438 PlaceholderRange(offset=0, length=3),
@@ -450,7 +462,7 @@ def test_hash_tokens_different_mm_input(hash_fn):
462 init_none_hash(hash_fn)
463
464 request1 = make_request(
453 - request_id=0,
465 + request_id="0",
466 prompt_token_ids=[_ for _ in range(6)],
467 mm_positions=[
468 PlaceholderRange(offset=0, length=3),
@@ -459,7 +471,7 @@ def test_hash_tokens_different_mm_input(hash_fn):
471 mm_hashes=["hash1", "hash2"],
472 )
473 request2 = make_request(
462 - request_id=1,
474 + request_id="1",
475 prompt_token_ids=[_ for _ in range(6)],
476 mm_positions=[
477 PlaceholderRange(offset=0, length=3),
@@ -479,7 +491,7 @@ def test_hash_request_tokens_no_mm_inputs(hash_fn):
491 init_none_hash(hash_fn)
492
493 request = make_request(
482 - request_id=0,
494 + request_id="0",
495 prompt_token_ids=[_ for _ in range(6)],
496 mm_positions=None,
497 mm_hashes=None,
@@ -844,7 +856,7 @@ def test_allocate_with_lookahead():
856 )
857
858 request = make_request(
847 - request_id=0,
859 + request_id="0",
860 prompt_token_ids=[],
861 mm_positions=None,
862 mm_hashes=None,
tests/v1/core/test_prefix_caching.py
+21 -10
@@ -9,7 +9,9 @@ import pytest
9 import torch
10
11 from vllm.distributed.kv_events import AllBlocksCleared, BlockRemoved
12 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
12 +from vllm.multimodal.inputs import (MultiModalBatchedField,
13 + MultiModalFieldElem, MultiModalKwargsItem,
14 + PlaceholderRange)
15 from vllm.sampling_params import SamplingParams
16 from vllm.utils import sha256, sha256_cbor_64bit
17 from vllm.v1.core.block_pool import BlockPool
@@ -21,21 +23,30 @@ from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
23 KVCacheGroupSpec, SlidingWindowSpec)
24
25
24 -def make_request(request_id,
25 - prompt_token_ids,
26 - mm_positions=None,
27 - mm_hashes=None,
28 - prompt_logprobs: Optional[int] = None,
29 - cache_salt: Optional[str] = None):
26 +def make_request(
27 + request_id: str,
28 + prompt_token_ids: list[int],
29 + mm_positions: Optional[list[PlaceholderRange]] = None,
30 + mm_hashes: Optional[list[str]] = None,
31 + prompt_logprobs: Optional[int] = None,
32 + cache_salt: Optional[str] = None,
33 +):
34 if mm_positions is None:
31 - multi_modal_inputs = None
35 + mm_kwargs = None
36 else:
33 - multi_modal_inputs = [MultiModalKwargs({})] * len(mm_positions)
37 + mm_elem = MultiModalFieldElem(
38 + modality="dummy_m",
39 + key="dummy_k",
40 + data=None,
41 + field=MultiModalBatchedField(),
42 + )
43 + mm_item = MultiModalKwargsItem.from_elems([mm_elem])
44 + mm_kwargs = [mm_item] * len(mm_positions)
45
46 return Request(
47 request_id=request_id,
48 prompt_token_ids=prompt_token_ids,
38 - multi_modal_inputs=multi_modal_inputs,
49 + multi_modal_kwargs=mm_kwargs,
50 multi_modal_hashes=mm_hashes,
51 multi_modal_placeholders=mm_positions,
52 sampling_params=SamplingParams(max_tokens=17,
tests/v1/core/test_scheduler.py
+15 -6
@@ -8,7 +8,9 @@ import torch
8
9 from vllm.config import (CacheConfig, KVTransferConfig, ModelConfig,
10 SchedulerConfig, SpeculativeConfig, VllmConfig)
11 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
11 +from vllm.multimodal.inputs import (MultiModalBatchedField,
12 + MultiModalFieldElem, MultiModalKwargsItem,
13 + PlaceholderRange)
14 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
15 from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
16 from vllm.v1.core.sched.scheduler import Scheduler
@@ -1304,7 +1306,7 @@ def create_requests_with_priority(
1306 priorities: list[int],
1307 arrival_times: Optional[list[float]] = None,
1308 num_tokens: int = 10,
1307 - mm_positions: Optional[list[PlaceholderRange]] = None,
1309 + mm_positions: Optional[list[list[PlaceholderRange]]] = None,
1310 max_tokens: int = 16,
1311 stop_token_ids: Optional[list[int]] = None,
1312 prompt_logprobs: Optional[int] = None):
@@ -1323,16 +1325,23 @@ def create_requests_with_priority(
1325 for i in range(num_requests):
1326 if mm_positions is not None:
1327 mm_position = mm_positions[i]
1326 - mm_inputs = [MultiModalKwargs({})] * len(mm_position)
1328 + mm_elem = MultiModalFieldElem(
1329 + modality="dummy_m",
1330 + key="dummy_k",
1331 + data=None,
1332 + field=MultiModalBatchedField(),
1333 + )
1334 + mm_item = MultiModalKwargsItem.from_elems([mm_elem])
1335 + mm_kwargs = [mm_item] * len(mm_position)
1336 else:
1337 mm_position = None
1329 - mm_inputs = None
1338 + mm_kwargs = None
1339 request = Request(
1340 request_id=f"{i}",
1341 prompt_token_ids=[i] * num_tokens,
1342 sampling_params=sampling_params,
1343 pooling_params=None,
1335 - multi_modal_inputs=mm_inputs,
1344 + multi_modal_kwargs=mm_kwargs,
1345 multi_modal_placeholders=mm_position,
1346 multi_modal_hashes=None,
1347 eos_token_id=EOS_TOKEN_ID,
@@ -1816,7 +1825,7 @@ def test_schedule_skip_tokenizer_init_structured_output_request():
1825 request = Request(
1826 request_id="0",
1827 prompt_token_ids=[0, 1],
1819 - multi_modal_inputs=None,
1828 + multi_modal_kwargs=None,
1829 multi_modal_hashes=None,
1830 multi_modal_placeholders=None,
1831 sampling_params=sampling_params,
tests/v1/core/utils.py
+14 -5
@@ -6,7 +6,9 @@ import torch
6
7 from vllm.config import (CacheConfig, KVTransferConfig, ModelConfig,
8 SchedulerConfig, SpeculativeConfig, VllmConfig)
9 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
9 +from vllm.multimodal.inputs import (MultiModalBatchedField,
10 + MultiModalFieldElem, MultiModalKwargsItem,
11 + PlaceholderRange)
12 from vllm.sampling_params import SamplingParams
13 from vllm.v1.core.sched.async_scheduler import AsyncScheduler
14 from vllm.v1.core.sched.scheduler import Scheduler
@@ -115,7 +117,7 @@ def create_scheduler(
117 def create_requests(
118 num_requests: int,
119 num_tokens: int = 10,
118 - mm_positions: Optional[list[PlaceholderRange]] = None,
120 + mm_positions: Optional[list[list[PlaceholderRange]]] = None,
121 max_tokens: int = 16,
122 stop_token_ids: Optional[list[int]] = None,
123 prompt_logprobs: Optional[int] = None,
@@ -129,10 +131,17 @@ def create_requests(
131 for i in range(num_requests):
132 if mm_positions is not None:
133 mm_position = mm_positions[i]
132 - mm_inputs = [MultiModalKwargs({})] * len(mm_position)
134 + mm_elem = MultiModalFieldElem(
135 + modality="dummy_m",
136 + key="dummy_k",
137 + data=None,
138 + field=MultiModalBatchedField(),
139 + )
140 + mm_item = MultiModalKwargsItem.from_elems([mm_elem])
141 + mm_kwargs = [mm_item] * len(mm_position)
142 else:
143 mm_position = None
135 - mm_inputs = None
144 + mm_kwargs = None
145 prompt_token_ids = ([0] * num_tokens if same_prompt else [i] *
146 num_tokens)
147 request = Request(
@@ -140,7 +149,7 @@ def create_requests(
149 prompt_token_ids=prompt_token_ids,
150 sampling_params=sampling_params,
151 pooling_params=None,
143 - multi_modal_inputs=mm_inputs,
152 + multi_modal_kwargs=mm_kwargs,
153 multi_modal_placeholders=mm_position,
154 multi_modal_hashes=None,
155 eos_token_id=EOS_TOKEN_ID,
tests/v1/engine/test_engine_core.py
+1 -1
@@ -35,7 +35,7 @@ def make_request() -> EngineCoreRequest:
35 return EngineCoreRequest(
36 request_id=str(uuid.uuid4()),
37 prompt_token_ids=PROMPT_TOKENS,
38 - mm_inputs=None,
38 + mm_kwargs=None,
39 mm_hashes=None,
40 mm_placeholders=None,
41 sampling_params=SamplingParams(),
tests/v1/engine/test_engine_core_client.py
+1 -1
@@ -52,7 +52,7 @@ def make_request(
52 return EngineCoreRequest(
53 request_id=str(uuid.uuid4()),
54 prompt_token_ids=prompt_tokens_ids,
55 - mm_inputs=None,
55 + mm_kwargs=None,
56 mm_hashes=None,
57 mm_placeholders=None,
58 sampling_params=params,
tests/v1/engine/test_output_processor.py
+5 -5
@@ -53,7 +53,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind,
53 EngineCoreRequest(request_id=f"request-{idx}",
54 prompt_token_ids=prompt_tokens,
55 arrival_time=0,
56 - mm_inputs=None,
56 + mm_kwargs=None,
57 mm_hashes=None,
58 mm_placeholders=None,
59 eos_token_id=None,
@@ -402,7 +402,7 @@ def test_logprobs_processor(request_output_kind: RequestOutputKind,
402 EngineCoreRequest(request_id=request_id_list[idx],
403 prompt_token_ids=prompt_tokens,
404 arrival_time=0,
405 - mm_inputs=None,
405 + mm_kwargs=None,
406 mm_hashes=None,
407 mm_placeholders=None,
408 eos_token_id=None,
@@ -567,7 +567,7 @@ def test_stop_token(include_stop_str_in_output: bool,
567 request_id=request_id,
568 prompt_token_ids=prompt_tokens,
569 arrival_time=0,
570 - mm_inputs=None,
570 + mm_kwargs=None,
571 mm_hashes=None,
572 mm_placeholders=None,
573 eos_token_id=eos_token_id,
@@ -666,7 +666,7 @@ def test_stop_string(include_stop_str_in_output: bool,
666 request_id=request_id_list[idx],
667 prompt_token_ids=prompt_tokens,
668 arrival_time=0,
669 - mm_inputs=None,
669 + mm_kwargs=None,
670 mm_hashes=None,
671 mm_placeholders=None,
672 eos_token_id=None,
@@ -782,7 +782,7 @@ def test_iteration_stats(dummy_test_vectors):
782 request_id=f"request-{idx}",
783 prompt_token_ids=prompt_tokens,
784 arrival_time=0,
785 - mm_inputs=None,
785 + mm_kwargs=None,
786 mm_hashes=None,
787 mm_placeholders=None,
788 eos_token_id=None,
tests/v1/kv_connector/unit/utils.py
+1 -1
@@ -154,7 +154,7 @@ def create_request(
154 prompt_token_ids=prompt_token_ids,
155 sampling_params=sampling_params,
156 pooling_params=None,
157 - multi_modal_inputs=None,
157 + multi_modal_kwargs=None,
158 multi_modal_placeholders=None,
159 multi_modal_hashes=None,
160 eos_token_id=EOS_TOKEN_ID,
tests/v1/tpu/worker/test_tpu_model_runner.py
+1 -1
@@ -64,7 +64,7 @@ def _schedule_new_request(*req_ids: str) -> SchedulerOutput:
64 NewRequestData(
65 req_id=req_id,
66 prompt_token_ids=[1, 2, 3],
67 - mm_inputs=[],
67 + mm_kwargs=[],
68 mm_hashes=[],
69 mm_positions=[],
70 sampling_params=SamplingParams(),
tests/v1/worker/test_gpu_input_batch.py
+1 -1
@@ -203,7 +203,7 @@ def _construct_cached_request_state(req_id_suffix: int):
203 prompt_token_ids=prompt_token_ids,
204 sampling_params=_create_sampling_params(),
205 pooling_params=None,
206 - mm_inputs=[],
206 + mm_kwargs=[],
207 mm_positions=[],
208 block_ids=([], ),
209 generator=None,
tests/v1/worker/test_gpu_model_runner.py
+1 -1
@@ -120,7 +120,7 @@ def _schedule_new_request(*req_ids: str) -> SchedulerOutput:
120 NewRequestData(
121 req_id=req_id,
122 prompt_token_ids=[1, 2, 3],
123 - mm_inputs=[],
123 + mm_kwargs=[],
124 mm_hashes=[],
125 mm_positions=[],
126 sampling_params=SamplingParams(),
vllm/multimodal/inputs.py
+120 -21
@@ -4,7 +4,7 @@
4 from abc import ABC, abstractmethod
5 from collections import UserDict, defaultdict
6 from collections.abc import Mapping, Sequence
7 -from dataclasses import dataclass
7 +from dataclasses import dataclass, replace
8 from functools import partial
9 from itertools import accumulate
10 from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,
@@ -198,7 +198,7 @@ A dictionary containing nested tensors which have been batched via
198 """
199
200
201 -@dataclass(frozen=True)
201 +@dataclass
202 class MultiModalFieldElem:
203 """
204 Represents a keyword argument corresponding to a multi-modal item
@@ -218,11 +218,14 @@ class MultiModalFieldElem:
218 i.e. the name of the keyword argument to be passed to the model.
219 """
220
221 - data: NestedTensors
221 + data: Optional[NestedTensors]
222 """
223 The tensor data of this field in
224 [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs],
225 i.e. the value of the keyword argument to be passed to the model.
226 +
227 + It may be set to `None` if it is determined that the item is cached
228 + in `EngineCore`.
229 """
230
231 field: "BaseMultiModalField"
@@ -235,8 +238,15 @@ class MultiModalFieldElem:
238 if not isinstance(other, self.__class__):
239 return False
240
241 + if self.data is None:
242 + data_equal = other.data is None
243 + elif other.data is None:
244 + data_equal = self.data is None
245 + else:
246 + data_equal = nested_tensors_equal(self.data, other.data)
247 +
248 return ((self.modality, self.key) == (other.modality, other.key)
239 - and nested_tensors_equal(self.data, other.data)
249 + and data_equal
250 and type(self.field) == type(other.field)) # noqa: E721
251
252
@@ -280,10 +290,20 @@ class BaseMultiModalField(ABC):
290 raise NotImplementedError
291
292 @abstractmethod
283 - def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
293 + def _reduce_data(
294 + self,
295 + batch: list[NestedTensors],
296 + *,
297 + pin_memory: bool,
298 + ) -> NestedTensors:
299 raise NotImplementedError
300
286 - def reduce_data(self, elems: list[MultiModalFieldElem]) -> NestedTensors:
301 + def reduce_data(
302 + self,
303 + elems: list[MultiModalFieldElem],
304 + *,
305 + pin_memory: bool = False,
306 + ) -> NestedTensors:
307 """
308 Merge the data from multiple instances of
309 [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem].
@@ -295,7 +315,13 @@ class BaseMultiModalField(ABC):
315 if len(set(field_types)) > 1:
316 raise ValueError(f"Cannot merge different {field_types=}")
317
298 - return self._reduce_data([item.data for item in elems])
318 + validated_data = list[NestedTensors]()
319 + for i, elem in enumerate(elems):
320 + assert elem.data is not None, (
321 + f"Cannot merge with empty `elems[{i}]`")
322 + validated_data.append(elem.data)
323 +
324 + return self._reduce_data(validated_data, pin_memory=pin_memory)
325
326
327 @dataclass(frozen=True)
@@ -314,7 +340,12 @@ class MultiModalBatchedField(BaseMultiModalField):
340 field_factory = self._field_factory(modality=modality, key=key)
341 return [field_factory(item) for item in data]
342
317 - def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
343 + def _reduce_data(
344 + self,
345 + batch: list[NestedTensors],
346 + *,
347 + pin_memory: bool,
348 + ) -> NestedTensors:
349 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
350 if len(batch) == 1:
351 # An optimization when `batch` contains only one tensor:
@@ -323,7 +354,11 @@ class MultiModalBatchedField(BaseMultiModalField):
354 return batch[0].unsqueeze(0).contiguous()
355 first_shape = batch[0].shape
356 if all(elem.shape == first_shape for elem in batch):
326 - return torch.stack(batch)
357 + out = torch.empty((len(batch), *batch[0].shape),
358 + dtype=batch[0].dtype,
359 + device=batch[0].device,
360 + pin_memory=pin_memory)
361 + return torch.stack(batch, out=out)
362
363 return batch
364
@@ -350,7 +385,12 @@ class MultiModalFlatField(BaseMultiModalField):
385 "torch.Tensor is required for multiple slices"
386 return [field_factory(data[cast(slice, s)]) for s in self.slices]
387
353 - def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
388 + def _reduce_data(
389 + self,
390 + batch: list[NestedTensors],
391 + *,
392 + pin_memory: bool,
393 + ) -> NestedTensors:
394 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
395 if len(batch) == 1:
396 # An optimization when `batch` contains only one tensor:
@@ -358,13 +398,21 @@ class MultiModalFlatField(BaseMultiModalField):
398 # - will achieve zero-copy if the tensor is contiguous
399 return batch[0].contiguous()
400
361 - def _expect_same_shape(tensor: torch.Tensor):
362 - return tensor.shape[:self.dim] + tensor.shape[self.dim + 1:]
401 + dim = self.dim + (self.dim < 0) * len(batch[0].shape)
402 +
403 + def _shape_before_after(tensor: torch.Tensor):
404 + return tensor.shape[:dim], tensor.shape[dim + 1:]
405
364 - first_shape = _expect_same_shape(batch[0])
406 + first_shape = _shape_before_after(batch[0])
407
366 - if all(_expect_same_shape(elem) == first_shape for elem in batch):
367 - return torch.concat(batch, dim=self.dim)
408 + if all(_shape_before_after(elem) == first_shape for elem in batch):
409 + shape_before, shape_after = first_shape
410 + shape_concat = sum(item.shape[dim] for item in batch)
411 + out = torch.empty((*shape_before, shape_concat, *shape_after),
412 + dtype=batch[0].dtype,
413 + device=batch[0].device,
414 + pin_memory=pin_memory)
415 + return torch.concat(batch, dim=self.dim, out=out)
416
417 assert self.dim == 0, "dim == 0 is required for nested list"
418 return [e for elem in batch for e in elem]
@@ -387,7 +435,12 @@ class MultiModalSharedField(BaseMultiModalField):
435 field_factory = self._field_factory(modality=modality, key=key)
436 return [field_factory(data)] * self.batch_size
437
390 - def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
438 + def _reduce_data(
439 + self,
440 + batch: list[NestedTensors],
441 + *,
442 + pin_memory: bool,
443 + ) -> NestedTensors:
444 return batch[0]
445
446
@@ -594,11 +647,53 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
647 def from_elems(elems: Sequence[MultiModalFieldElem]):
648 return MultiModalKwargsItem({elem.key: elem for elem in elems})
649
597 - @property
598 - def modality(self) -> str:
650 + def __init__(self, data: Mapping[str, MultiModalFieldElem]) -> None:
651 + super().__init__(data)
652 +
653 modalities = {elem.modality for elem in self.data.values()}
654 assert len(modalities) == 1, f"Found different modalities={modalities}"
601 - return next(iter(modalities))
655 + self._modality = next(iter(modalities))
656 +
657 + self._is_empty = any(elem.data is None for elem in self.values())
658 +
659 + @property
660 + def modality(self) -> str:
661 + return self._modality
662 +
663 + @property
664 + def is_empty(self) -> bool:
665 + return self._is_empty
666 +
667 + def get_data(self) -> Optional[Mapping[str, NestedTensors]]:
668 + if self._is_empty:
669 + return None
670 +
671 + out_data = dict[str, NestedTensors]()
672 + for key, elem in self.items():
673 + assert elem.data is not None, (
674 + f"Cannot get data of empty `elem[{key!r}]`")
675 + out_data[key] = elem.data
676 +
677 + return out_data
678 +
679 + def require_data(self) -> Mapping[str, NestedTensors]:
680 + if (data := self.get_data()) is None:
681 + raise RuntimeError("Cannot get data of empty item")
682 +
683 + return data
684 +
685 + # These methods create a new item to avoid mutating cached items in place
686 + def with_data(self, data: Mapping[str, NestedTensors]):
687 + return MultiModalKwargsItem({
688 + key: replace(elem, data=data[key])
689 + for key, elem in self.items()
690 + })
691 +
692 + def without_data(self):
693 + return MultiModalKwargsItem({
694 + key: replace(elem, data=None)
695 + for key, elem in self.items()
696 + })
697
698
699 # NOTE: UserDict is for V0 compatibility.
@@ -650,7 +745,11 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
745 return MultiModalKwargs.from_items(items)
746
747 @staticmethod
653 - def from_items(items: Sequence[MultiModalKwargsItem]):
748 + def from_items(
749 + items: Sequence[MultiModalKwargsItem],
750 + *,
751 + pin_memory: bool = False,
752 + ):
753 """Construct a new
754 [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs]
755 from multiple items."""
@@ -660,7 +759,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
759 elems_by_key[key].append(elem)
760
761 data = {
663 - key: elems[0].field.reduce_data(elems)
762 + key: elems[0].field.reduce_data(elems, pin_memory=pin_memory)
763 for key, elems in elems_by_key.items() if len(elems) > 0
764 }
765
vllm/multimodal/utils.py
+65 -68
@@ -3,6 +3,7 @@
3
4 import asyncio
5 import atexit
6 +from collections.abc import Iterable
7 from concurrent.futures import ThreadPoolExecutor
8 from itertools import groupby
9 from pathlib import Path
@@ -13,6 +14,7 @@ import numpy as np
14 import numpy.typing as npt
15 import torch
16 from PIL import Image, UnidentifiedImageError
17 +from typing_extensions import deprecated
18
19 import vllm.envs as envs
20 from vllm.connections import HTTPConnection, global_http_connection
@@ -23,17 +25,17 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
25 from .audio import AudioMediaIO
26 from .base import MediaIO
27 from .image import ImageEmbeddingMediaIO, ImageMediaIO
26 -from .inputs import PlaceholderRange
28 from .video import VideoMediaIO
29
30 _M = TypeVar("_M")
31
32 if TYPE_CHECKING:
32 - from .hasher import MultiModalHashDict
33 - from .inputs import MultiModalKwargs, MultiModalPlaceholderDict
33 + from .inputs import (BatchedTensorInputs, MultiModalKwargs,
34 + MultiModalKwargsItem, MultiModalPlaceholderDict)
35 else:
35 - MultiModalHashDict = Any
36 + BatchedTensorInputs = Any
37 MultiModalKwargs = Any
38 + MultiModalKwargsItem = Any
39 MultiModalPlaceholderDict = Any
40
41 global_thread_pool = ThreadPoolExecutor(
@@ -331,79 +333,32 @@ def encode_video_base64(frames: npt.NDArray) -> str:
333 return video_io.encode_base64(frames)
334
335
334 -def merge_and_sort_multimodal_metadata(
335 - mm_positions: MultiModalPlaceholderDict,
336 - mm_hashes: Optional[MultiModalHashDict],
337 -) -> tuple[list[str], list[PlaceholderRange], Optional[list[str]]]:
338 - """Given a MultiModalPlaceholderDict, merge all PlaceholderRange
339 - objects from all available modalities into a single list of
340 - PlaceholderRange, sorted by their offset (starting index in the input
341 - sequence) in the ascending order.
336 +def argsort_mm_positions(
337 + mm_positions: MultiModalPlaceholderDict) -> list[tuple[str, int]]:
338 + """
339 + Given a `MultiModalPlaceholderDict`, output a sequence of keys to
340 + sort the dictionary by `offset` (starting index in the input sequence)
341 + in ascending order.
342
343 - Optionally if a `MultiModalHashDict` is given, same operation will be
344 - applied to the object and the sorted list of hashes will be returned.
345 -
343 Returns:
347 - list[str]: List of item modalities in order of their positions in the
348 - input sequence.
349 - list[PlaceholderRange]: Sorted list of all PlaceholderRanges from
350 - mm_positions.
351 - Optional[list[str]]: Sorted list of all hashes from mm_hashes if given,
352 - None otherwise.
344 + A list of `(modality, idx)`, which can be used to access an item
345 + by `mm_positions[modality][idx]`.
346 """
347 + flat_items = ((modality, idx, item)
348 + for modality, items in mm_positions.items()
349 + for idx, item in enumerate(items))
350
355 - modalities = list(mm_positions.keys())
356 -
357 - assert len(modalities) > 0, "No modalities found in the mm_positions."
358 -
359 - # For single modality, placeholder ranges and hashes are already sorted
360 - # so we can return the list directly.
361 - if len(modalities) == 1:
362 - modality = modalities[0]
363 - placeholder_list = list(mm_positions[modality])
364 -
365 - return [modality] * len(
366 - placeholder_list
367 - ), placeholder_list, None if not mm_hashes else mm_hashes[modality]
368 -
369 - # Create a list of (modality, placeholder, hash) tuples for all placeholders
370 - all_items = []
371 - for modality in modalities:
372 - placeholder_list = list(mm_positions[modality])
373 - hash_list: list[Optional[str]] = list(
374 - mm_hashes[modality]) if mm_hashes and modality in mm_hashes else [
375 - None
376 - ] * len(placeholder_list)
377 -
378 - for placeholder, hash_value in zip(placeholder_list, hash_list):
379 - all_items.append((modality, placeholder, hash_value))
351 + sorted_flat_items = sorted(flat_items, key=lambda x: x[2].offset)
352
381 - # Sort all items by offset
382 - all_items.sort(key=lambda x: x[1].offset)
383 -
384 - # Split into separate lists
385 - sorted_modalities = [item[0] for item in all_items]
386 - merged_placeholders = [item[1] for item in all_items]
387 - merged_hashes = [str(item[2])
388 - for item in all_items] if mm_hashes is not None else None
389 -
390 - return sorted_modalities, merged_placeholders, merged_hashes
353 + return [(modality, idx) for modality, idx, _ in sorted_flat_items]
354
355
356 +# Temporary back-compatibility for plugins that define model runner
357 +@deprecated("`group_mm_inputs_by_modality` is superseded by "
358 + "`group_mm_kwargs_by_modality` and will be removed in v0.13. "
359 + "Please use `group_mm_kwargs_by_modality` instead.")
360 def group_mm_inputs_by_modality(
361 mm_inputs: list[MultiModalKwargs]) -> list[list[MultiModalKwargs]]:
395 - """Group consecutive MultiModalKwargs from mm_inputs with the same modality
396 - together into the same list for batching purpose. For MultiModalKwargs with
397 - multiple modalities, put them into their own list.
398 -
399 - Args:
400 - mm_inputs: List of MultiModalKwargs.
401 -
402 - Returns:
403 - list[list[vllm.multimodal.MultiModalKwargs]]: List of list of
404 - `MultiModalKwargs`, each inner list contains consecutive
405 - `MultiModalKwargs` with same modality.
406 - """
362 if not mm_inputs:
363 return []
364
@@ -426,6 +381,48 @@ def group_mm_inputs_by_modality(
381 ]
382
383
384 +def group_mm_kwargs_by_modality(
385 + mm_kwargs: list[MultiModalKwargsItem],
386 + *,
387 + device: torch.types.Device = None,
388 + pin_memory: bool = False,
389 +) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
390 + """Group consecutive `MultiModalKwargsItem`s from `mm_kwargs` with the same
391 + modality together into the same `MultiModalKwargs` instance.
392 +
393 + Args:
394 + mm_inputs: List of `MultiModalKwargsItem`.
395 +
396 + Yields:
397 + A tuple `(modality, num_items, grouped_kwargs)`.
398 + """
399 + from vllm.multimodal.inputs import MultiModalKwargs
400 +
401 + for modality, items in groupby(mm_kwargs, key=lambda item: item.modality):
402 + items_lst = list(items)
403 +
404 + # mm_kwargs_group = MultiModalKwargs.from_items(items_lst,
405 + # pin_memory=pin_memory)
406 +
407 + # if device is not None:
408 + # mm_kwargs_group = json_map_leaves(lambda x: x.to(device=device),
409 + # mm_kwargs_group.data)
410 +
411 + # TODO: Once V0 is removed, we can use the merging logic above
412 + # to avoid creating an extra batch dimension (except for fields
413 + # that are meant to be stacked anyway).
414 + # We will also need to update each model to remove `flatten_bn`.
415 + mm_kwargs_group = MultiModalKwargs.as_kwargs(
416 + MultiModalKwargs.batch(
417 + [MultiModalKwargs.from_items([item]) for item in items_lst],
418 + pin_memory=pin_memory,
419 + ),
420 + device=device,
421 + )
422 +
423 + yield modality, len(items_lst), mm_kwargs_group
424 +
425 +
426 def run_dp_sharded_vision_model(image_input: torch.Tensor,
427 vision_model: torch.nn.Module) -> torch.Tensor:
428 """Run a vision model with data parallelism (DP) sharding. The function
vllm/v1/core/sched/output.py
+5 -5
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
13 from vllm.distributed.kv_transfer.kv_connector.v1.base import (
14 KVConnectorMetadata)
15 from vllm.lora.request import LoRARequest
16 - from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
16 + from vllm.multimodal.inputs import MultiModalKwargsItem, PlaceholderRange
17 from vllm.pooling_params import PoolingParams
18 from vllm.sampling_params import SamplingParams
19 from vllm.v1.request import Request
@@ -24,7 +24,7 @@ class NewRequestData:
24
25 req_id: str
26 prompt_token_ids: list[int]
27 - mm_inputs: list[MultiModalKwargs]
27 + mm_kwargs: list[MultiModalKwargsItem]
28 mm_hashes: list[str]
29 mm_positions: list[PlaceholderRange]
30 sampling_params: Optional[SamplingParams]
@@ -42,7 +42,7 @@ class NewRequestData:
42 return cls(
43 req_id=request.request_id,
44 prompt_token_ids=request.prompt_token_ids,
45 - mm_inputs=request.mm_inputs,
45 + mm_kwargs=request.mm_kwargs,
46 mm_hashes=request.mm_hashes,
47 mm_positions=request.mm_positions,
48 sampling_params=request.sampling_params,
@@ -56,7 +56,7 @@ class NewRequestData:
56 return (f"NewRequestData("
57 f"req_id={self.req_id},"
58 f"prompt_token_ids={self.prompt_token_ids},"
59 - f"mm_inputs={self.mm_inputs},"
59 + f"mm_kwargs={self.mm_kwargs},"
60 f"mm_hashes={self.mm_hashes},"
61 f"mm_positions={self.mm_positions},"
62 f"sampling_params={self.sampling_params},"
@@ -70,7 +70,7 @@ class NewRequestData:
70 return (f"NewRequestData("
71 f"req_id={self.req_id},"
72 f"prompt_token_ids_len={len(self.prompt_token_ids)},"
73 - f"mm_inputs={self.mm_inputs},"
73 + f"mm_kwargs={self.mm_kwargs},"
74 f"mm_hashes={self.mm_hashes},"
75 f"mm_positions={self.mm_positions},"
76 f"sampling_params={self.sampling_params},"
vllm/v1/engine/__init__.py
+2 -4
@@ -3,15 +3,13 @@
3
4 import enum
5 import time
6 -from collections.abc import Sequence
6 from typing import Any, Optional, Union
7
8 import msgspec
9 import torch
10
11 from vllm.lora.request import LoRARequest
13 -from vllm.multimodal import MultiModalKwargs
14 -from vllm.multimodal.inputs import PlaceholderRange
12 +from vllm.multimodal.inputs import MultiModalKwargsItem, PlaceholderRange
13 from vllm.pooling_params import PoolingParams
14 from vllm.sampling_params import SamplingParams
15 from vllm.v1.metrics.stats import SchedulerStats
@@ -49,7 +47,7 @@ class EngineCoreRequest(
47
48 request_id: str
49 prompt_token_ids: list[int]
52 - mm_inputs: Optional[Sequence[Optional[MultiModalKwargs]]]
50 + mm_kwargs: Optional[list[MultiModalKwargsItem]]
51 mm_hashes: Optional[list[str]]
52 mm_placeholders: Optional[list[PlaceholderRange]]
53 sampling_params: Optional[SamplingParams]
vllm/v1/engine/core.py
+4 -3
@@ -409,12 +409,13 @@ class EngineCore:
409 request initialization running in parallel with Model forward
410 """
411 if request.mm_hashes is not None:
412 - assert request.mm_inputs is not None
412 + assert request.mm_kwargs is not None
413 +
414 # Note on thread safety: no race condition.
415 # `mm_input_cache_server` is reset at the end of LLMEngine init,
416 # and will only accessed in the input processing thread afterwards.
416 - request.mm_inputs = self.mm_input_cache_server.get_and_update(
417 - request.mm_inputs, request.mm_hashes)
417 + request.mm_kwargs = self.mm_input_cache_server.get_and_update(
418 + request.mm_kwargs, request.mm_hashes)
419
420 req = Request.from_engine_core_request(request)
421 if req.use_structured_output:
vllm/v1/engine/mm_input_cache.py
+37 -41
@@ -1,11 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from collections.abc import Sequence
4 -from typing import TYPE_CHECKING, Optional
3 +from collections.abc import Mapping
4 +from typing import TYPE_CHECKING
5
6 -from vllm.multimodal import MultiModalKwargs, MultiModalRegistry
6 +from vllm.multimodal import MultiModalRegistry
7 from vllm.multimodal.cache import MultiModalCache, MultiModalCacheItemMetadata
8 -from vllm.utils import is_list_of
8 +from vllm.multimodal.inputs import MultiModalKwargsItem, NestedTensors
9
10 if TYPE_CHECKING:
11 from vllm.config import ModelConfig
@@ -17,23 +17,23 @@ if TYPE_CHECKING:
17 # -- P0:
18 # - BaseMultiModalProcessor calls MultiModalHasher to get the `mm_hash` of
19 # each input multi-modal item (e.g. image),
20 -# - BaseMultiModalProcessor processes the input items into `mm_inputs`,
20 +# - BaseMultiModalProcessor processes the input items into `mm_kwargs`,
21 # which are MultiModalKwargsItem instances that each correspond to an
22 # input multi-modal item.
23 -# - MultiModalInputCacheClient accepts the `mm_inputs` and corresponding
23 +# - MultiModalInputCacheClient accepts the `mm_kwargs` and corresponding
24 # `mm_hash` for each item. It stores the `mm_hash` as keys and the size
25 -# of `mm_inputs`, but not the `mm_inputs` themselves, to avoid taking
25 +# of `mm_kwargs`, but not the `mm_kwargs` themselves, to avoid taking
26 # up additional memory in P0.
27 # - The `mm_hash` is always sent to P1.
28 -# - The corresponding `mm_inputs` are only sent to P1 if they are not cached
28 +# - The corresponding `mm_kwargs` are only sent to P1 if they are not cached
29 # in MultiModalInputCacheServer.
30 #
31 # -- P1:
32 -# - If the `mm_hash` is cached (i.e. `mm_inputs` are not sent from P0),
33 -# MultiModalInputCacheServer retrieves the corresponding `mm_inputs`.
34 -# - If the `mm_hash` is not cached (i.e. `mm_inputs` are sent from P0),
35 -# MultiModalInputCacheServer stores `mm_inputs` under the key `mm_hash`.
36 -# - Either way, the `mm_hash` and corresponding `mm_inputs` are sent to
32 +# - If the `mm_hash` is cached (i.e. `mm_kwargs` are not sent from P0),
33 +# MultiModalInputCacheServer retrieves the corresponding `mm_kwargs`.
34 +# - If the `mm_hash` is not cached (i.e. `mm_kwargs` are sent from P0),
35 +# MultiModalInputCacheServer stores `mm_kwargs` under the key `mm_hash`.
36 +# - Either way, the `mm_hash` and corresponding `mm_kwargs` are sent to
37 # the engine for model execution.
38 #
39 # Both Client and Server must perform cache update and eviction based on the
@@ -58,26 +58,24 @@ class MultiModalInputCacheClient:
58
59 def get_and_update(
60 self,
61 - mm_inputs: Sequence[MultiModalKwargs],
61 + mm_kwargs: list[MultiModalKwargsItem],
62 mm_hashes: list[str],
63 - ) -> Sequence[Optional[MultiModalKwargs]]:
64 - assert len(mm_inputs) == len(mm_hashes)
65 -
63 + ) -> list[MultiModalKwargsItem]:
64 if not self.enabled:
67 - assert is_list_of(mm_inputs, MultiModalKwargs)
68 - return mm_inputs
65 + return mm_kwargs
66 +
67 + assert len(mm_kwargs) == len(mm_hashes)
68
70 - full_mm_inputs = list[Optional[MultiModalKwargs]]()
71 - for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
69 + out_mm_items = list[MultiModalKwargsItem]()
70 + for mm_item, mm_hash in zip(mm_kwargs, mm_hashes):
71 if self.mm_cache.get(mm_hash) is not None:
73 - mm_input = None
72 + out_mm_items.append(mm_item.without_data())
73 else:
74 self.mm_cache[mm_hash] = \
76 - MultiModalCacheItemMetadata.wraps(mm_input)
77 -
78 - full_mm_inputs.append(mm_input)
75 + MultiModalCacheItemMetadata.wraps(mm_item.require_data())
76 + out_mm_items.append(mm_item)
77
80 - return full_mm_inputs
78 + return out_mm_items
79
80 def reset(self) -> None:
81 self.mm_cache.clear()
@@ -93,30 +91,28 @@ class MultiModalInputCacheServer:
91 self.enabled = mm_registry.enable_mm_input_cache(model_config)
92 self.mm_cache = MultiModalCache.get_lru_cache(
93 model_config.get_mm_input_cache_gb(),
96 - MultiModalKwargs,
94 + Mapping[str, NestedTensors],
95 )
96
97 def get_and_update(
98 self,
101 - mm_inputs: Sequence[Optional[MultiModalKwargs]],
99 + mm_kwargs: list[MultiModalKwargsItem],
100 mm_hashes: list[str],
103 - ) -> Sequence[MultiModalKwargs]:
104 - assert len(mm_inputs) == len(mm_hashes)
105 -
101 + ) -> list[MultiModalKwargsItem]:
102 if not self.enabled:
107 - assert is_list_of(mm_inputs, MultiModalKwargs)
108 - return mm_inputs
103 + return mm_kwargs
104
110 - full_mm_inputs = list[MultiModalKwargs]()
111 - for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
112 - if mm_input is None:
113 - mm_input = self.mm_cache[mm_hash]
114 - else:
115 - self.mm_cache[mm_hash] = mm_input
105 + assert len(mm_kwargs) == len(mm_hashes)
106
117 - full_mm_inputs.append(mm_input)
107 + out_mm_items = list[MultiModalKwargsItem]()
108 + for mm_item, mm_hash in zip(mm_kwargs, mm_hashes):
109 + if (mm_data := mm_item.get_data()) is None:
110 + out_mm_items.append(mm_item.with_data(self.mm_cache[mm_hash]))
111 + else:
112 + self.mm_cache[mm_hash] = mm_data
113 + out_mm_items.append(mm_item)
114
119 - return full_mm_inputs
115 + return out_mm_items
116
117 def reset(self) -> None:
118 self.mm_cache.clear()
vllm/v1/engine/processor.py
+25 -41
@@ -2,7 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import time
5 -from collections.abc import Mapping, Sequence
5 +from collections.abc import Mapping
6 from typing import Any, Literal, Optional, Union
7
8 from vllm.config import VllmConfig
@@ -10,11 +10,10 @@ from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
10 from vllm.inputs.parse import split_enc_dec_inputs
11 from vllm.inputs.preprocess import InputPreprocessor
12 from vllm.lora.request import LoRARequest
13 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
14 - MultiModalRegistry)
15 -from vllm.multimodal.inputs import PlaceholderRange
13 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
14 +from vllm.multimodal.inputs import MultiModalKwargsItem, PlaceholderRange
15 from vllm.multimodal.processing import EncDecMultiModalProcessor
17 -from vllm.multimodal.utils import merge_and_sort_multimodal_metadata
16 +from vllm.multimodal.utils import argsort_mm_positions
17 from vllm.pooling_params import PoolingParams
18 from vllm.sampling_params import SamplingParams
19 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -296,57 +295,42 @@ class Processor:
295 pooling_params = params.clone()
296
297 # Multimodal related.
299 - sorted_mm_inputs: Optional[Sequence[Optional[MultiModalKwargs]]] = None
298 + sorted_mm_inputs: Optional[list[MultiModalKwargsItem]] = None
299 sorted_mm_positions: Optional[list[PlaceholderRange]] = None
300 sorted_mm_hashes: Optional[list[str]] = None
301 if decoder_inputs["type"] == "multimodal":
302 decoder_mm_inputs = decoder_inputs["mm_kwargs"]
303 + decoder_mm_positions = decoder_inputs["mm_placeholders"]
304 + decoder_mm_hashes = decoder_inputs.get("mm_hashes")
305
306 # Merge and flatten multimodal placeholders, hashes and inputs
307 # from dictionaries to lists, and sort them by each item's position
308 # in the input sequence.
308 - (
309 - sorted_item_modalities,
310 - sorted_mm_positions,
311 - sorted_mm_hashes,
312 - ) = merge_and_sort_multimodal_metadata(
313 - decoder_inputs["mm_placeholders"],
314 - decoder_inputs["mm_hashes"] if return_mm_hashes else None,
315 - )
316 -
317 - # The output of merged multi-modal processor (`decoder_mm_inputs`)
318 - # is a single MultiModalKwargs for all items from all modalities.
319 - # This code flattens kwargs for individual items in a list and
320 - # sorts them by each item's position in the input sequence if there
321 - # are multiple modalities.
322 - unique_modalities = set(sorted_item_modalities)
323 - if len(unique_modalities) > 1:
324 - orig_sorted_mm_inputs = []
325 - used_indices = {modality: 0 for modality in unique_modalities}
326 -
327 - for modality in sorted_item_modalities:
328 - items = decoder_mm_inputs.get_items(modality)
329 - item = items[used_indices[modality]]
330 -
331 - orig_sorted_mm_inputs.append(
332 - MultiModalKwargs.from_items([item]))
333 - used_indices[modality] += 1
334 - else:
335 - orig_sorted_mm_inputs = [
336 - MultiModalKwargs.from_items([item]) for item in
337 - decoder_mm_inputs.get_items(sorted_item_modalities[0])
338 - ]
309 + sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)
310 +
311 + sorted_mm_inputs = [
312 + decoder_mm_inputs.get_item(modality, idx)
313 + for modality, idx in sorted_mm_idxs
314 + ]
315 + sorted_mm_positions = [
316 + decoder_mm_positions[modality][idx]
317 + for modality, idx in sorted_mm_idxs
318 + ]
319 + sorted_mm_hashes = None if decoder_mm_hashes is None else [
320 + decoder_mm_hashes[modality][idx]
321 + for modality, idx in sorted_mm_idxs
322 + ]
323
324 if sorted_mm_hashes is not None:
325 sorted_mm_inputs = self.mm_input_cache_client.get_and_update(
342 - orig_sorted_mm_inputs, sorted_mm_hashes)
343 - else:
344 - sorted_mm_inputs = orig_sorted_mm_inputs
326 + sorted_mm_inputs,
327 + sorted_mm_hashes,
328 + )
329
330 return decoder_inputs.get("prompt"), EngineCoreRequest(
331 request_id=request_id,
332 prompt_token_ids=decoder_inputs["prompt_token_ids"],
349 - mm_inputs=sorted_mm_inputs,
333 + mm_kwargs=sorted_mm_inputs,
334 mm_hashes=sorted_mm_hashes,
335 mm_placeholders=sorted_mm_positions,
336 sampling_params=sampling_params,
vllm/v1/request.py
+10 -11
@@ -5,7 +5,7 @@ import enum
5 import time
6 from typing import TYPE_CHECKING, Any, Optional, Union
7
8 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
8 +from vllm.multimodal.inputs import MultiModalKwargsItem, PlaceholderRange
9 from vllm.pooling_params import PoolingParams
10 from vllm.sampling_params import SamplingParams
11 from vllm.utils import is_list_of
@@ -24,7 +24,7 @@ class Request:
24 self,
25 request_id: str,
26 prompt_token_ids: list[int],
27 - multi_modal_inputs: Optional[list[MultiModalKwargs]],
27 + multi_modal_kwargs: Optional[list[MultiModalKwargsItem]],
28 multi_modal_hashes: Optional[list[str]],
29 multi_modal_placeholders: Optional[list[PlaceholderRange]],
30 sampling_params: Optional[SamplingParams],
@@ -84,15 +84,15 @@ class Request:
84
85 # Multi-modal related
86 self.mm_positions = multi_modal_placeholders or []
87 - self.mm_inputs = multi_modal_inputs or []
87 + self.mm_kwargs = multi_modal_kwargs or []
88 self.mm_hashes: list[str] = multi_modal_hashes or []
89 - self.num_encoder_inputs = len(self.mm_inputs)
89 + self.num_encoder_inputs = len(self.mm_kwargs)
90 self.has_encoder_inputs = self.num_encoder_inputs > 0
91
92 # Sanity check
93 - assert len(self.mm_inputs) == len(self.mm_positions)
93 + assert len(self.mm_kwargs) == len(self.mm_positions)
94 if self.mm_hashes:
95 - assert len(self.mm_inputs) == len(self.mm_hashes)
95 + assert len(self.mm_kwargs) == len(self.mm_hashes)
96
97 # Read-only views
98 # Prevent directly appending to these lists since
@@ -110,16 +110,15 @@ class Request:
110
111 @classmethod
112 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
113 - if request.mm_inputs is not None:
114 - assert isinstance(request.mm_inputs, list)
115 - assert is_list_of(request.mm_inputs, MultiModalKwargs), (
116 - "mm_inputs was not updated in EngineCore.add_request")
113 + if request.mm_kwargs is not None:
114 + assert is_list_of(request.mm_kwargs, MultiModalKwargsItem), (
115 + "mm_kwargs was not updated in EngineCore.add_request")
116
117 return cls(
118 request_id=request.request_id,
119 client_index=request.client_index,
120 prompt_token_ids=request.prompt_token_ids,
122 - multi_modal_inputs=request.mm_inputs,
121 + multi_modal_kwargs=request.mm_kwargs,
122 multi_modal_hashes=request.mm_hashes,
123 multi_modal_placeholders=request.mm_placeholders,
124 sampling_params=request.sampling_params,
vllm/v1/serial_utils.py
+34 -14
@@ -113,6 +113,9 @@ class MsgpackEncoder:
113 int(v) if v is not None else None
114 for v in (obj.start, obj.stop, obj.step))
115
116 + if isinstance(obj, MultiModalKwargsItem):
117 + return self._encode_mm_item(obj)
118 +
119 if isinstance(obj, MultiModalKwargs):
120 mm: MultiModalKwargs = obj
121 if not mm.modalities:
@@ -120,17 +123,12 @@ class MsgpackEncoder:
123 return dict(mm)
124
125 # ignore the main dict, it will be re-indexed.
123 - # Encode a list of MultiModalKwargsItems as plain dicts
124 - # + special handling for .field.
126 # Any tensors *not* indexed by modality will be ignored.
126 - return [[{
127 - "modality": elem.modality,
128 - "key": elem.key,
129 - "data": self._encode_nested_tensors(elem.data),
130 - "field": self._encode_mm_field(elem.field),
131 - } for elem in item.values()]
132 - for itemlist in mm._items_by_modality.values()
133 - for item in itemlist]
127 + return [
128 + self._encode_mm_item(item)
129 + for itemlist in mm._items_by_modality.values()
130 + for item in itemlist
131 + ]
132
133 if isinstance(obj, UtilityResult):
134 result = obj.result
@@ -192,6 +190,23 @@ class MsgpackEncoder:
190 dtype = str(obj.dtype).removeprefix("torch.")
191 return dtype, obj.shape, data
192
193 + def _encode_mm_item(self,
194 + item: MultiModalKwargsItem) -> list[dict[str, Any]]:
195 + return [self._encode_mm_field_elem(elem) for elem in item.values()]
196 +
197 + def _encode_mm_field_elem(self,
198 + elem: MultiModalFieldElem) -> dict[str, Any]:
199 + return {
200 + "modality":
201 + elem.modality,
202 + "key":
203 + elem.key,
204 + "data": (None if elem.data is None else
205 + self._encode_nested_tensors(elem.data)),
206 + "field":
207 + self._encode_mm_field(elem.field),
208 + }
209 +
210 def _encode_nested_tensors(self, nt: NestedTensors) -> Any:
211 if isinstance(nt, torch.Tensor):
212 return self._encode_tensor(nt)
@@ -250,6 +265,8 @@ class MsgpackDecoder:
265 return self._decode_tensor(obj)
266 if t is slice:
267 return slice(*obj)
268 + if issubclass(t, MultiModalKwargsItem):
269 + return self._decode_mm_item(obj)
270 if issubclass(t, MultiModalKwargs):
271 if isinstance(obj, list):
272 return MultiModalKwargs.from_items(
@@ -311,15 +328,18 @@ class MsgpackDecoder:
328 # Convert back to proper shape & type
329 return arr.view(torch_dtype).view(shape)
330
314 - def _decode_mm_items(self, obj: list) -> list[MultiModalKwargsItem]:
331 + def _decode_mm_items(self, obj: list[Any]) -> list[MultiModalKwargsItem]:
332 return [self._decode_mm_item(v) for v in obj]
333
317 - def _decode_mm_item(self, obj: list) -> MultiModalKwargsItem:
334 + def _decode_mm_item(self, obj: list[Any]) -> MultiModalKwargsItem:
335 return MultiModalKwargsItem.from_elems(
336 [self._decode_mm_field_elem(v) for v in obj])
337
321 - def _decode_mm_field_elem(self, obj: dict) -> MultiModalFieldElem:
322 - obj["data"] = self._decode_nested_tensors(obj["data"])
338 + def _decode_mm_field_elem(self, obj: dict[str,
339 + Any]) -> MultiModalFieldElem:
340 + if obj["data"] is not None:
341 + obj["data"] = self._decode_nested_tensors(obj["data"])
342 +
343 # Reconstruct the field processor using MultiModalFieldConfig
344 factory_meth_name, *field_args = obj["field"]
345 factory_meth = getattr(MultiModalFieldConfig, factory_meth_name)
vllm/v1/worker/gpu_input_batch.py
+11 -2
@@ -7,9 +7,11 @@ from typing import Optional, cast
7
8 import numpy as np
9 import torch
10 +from typing_extensions import deprecated
11
12 from vllm.lora.request import LoRARequest
12 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
13 +from vllm.multimodal.inputs import (MultiModalKwargs, MultiModalKwargsItem,
14 + PlaceholderRange)
15 from vllm.pooling_params import PoolingParams
16 from vllm.sampling_params import SamplingParams, SamplingType
17 from vllm.utils import swap_dict_values
@@ -29,7 +31,7 @@ class CachedRequestState:
31
32 req_id: str
33 prompt_token_ids: list[int]
32 - mm_inputs: list[MultiModalKwargs]
34 + mm_kwargs: list[MultiModalKwargsItem]
35 mm_positions: list[PlaceholderRange]
36 sampling_params: Optional[SamplingParams]
37 pooling_params: Optional[PoolingParams]
@@ -51,6 +53,13 @@ class CachedRequestState:
53 def num_tokens(self) -> int:
54 return self.num_prompt_tokens + len(self.output_token_ids)
55
56 + # Temporary back-compatibility for plugins that define model runner
57 + @property
58 + @deprecated("`mm_inputs` is superseded by `mm_kwargs` and will be "
59 + "removed in v0.13. Please use `mm_kwargs` instead.")
60 + def mm_inputs(self) -> list[MultiModalKwargs]:
61 + return [MultiModalKwargs.from_items([item]) for item in self.mm_kwargs]
62 +
63 def get_token_id(self, idx: int) -> int:
64 if idx < self.num_prompt_tokens:
65 return self.prompt_token_ids[idx]
vllm/v1/worker/gpu_model_runner.py
+48 -49
@@ -40,9 +40,9 @@ from vllm.model_executor.models.interfaces import (is_mixture_of_experts,
40 from vllm.model_executor.models.interfaces_base import (
41 VllmModelForPooling, is_pooling_model, is_text_generation_model)
42 from vllm.multimodal import MULTIMODAL_REGISTRY
43 -from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargs,
43 +from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargsItem,
44 PlaceholderRange)
45 -from vllm.multimodal.utils import group_mm_inputs_by_modality
45 +from vllm.multimodal.utils import group_mm_kwargs_by_modality
46 from vllm.pooling_params import PoolingParams
47 from vllm.sampling_params import SamplingType
48 from vllm.sequence import IntermediateTensors, PoolerOutput
@@ -478,7 +478,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
478 self.requests[req_id] = CachedRequestState(
479 req_id=req_id,
480 prompt_token_ids=new_req_data.prompt_token_ids,
481 - mm_inputs=new_req_data.mm_inputs,
481 + mm_kwargs=new_req_data.mm_kwargs,
482 mm_positions=new_req_data.mm_positions,
483 sampling_params=sampling_params,
484 pooling_params=pooling_params,
@@ -496,18 +496,19 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
496 second_per_grid_ts = []
497 audio_feature_lengths = []
498 use_audio_in_video = False
499 - for mm_input in self.requests[req_id].mm_inputs:
499 + for item in self.requests[req_id].mm_kwargs:
500 + mm_input = item.require_data()
501 if mm_input.get("image_grid_thw") is not None:
501 - image_grid_thw.extend(
502 + image_grid_thw.append(
503 mm_input["image_grid_thw"].tolist())
504 if mm_input.get("video_grid_thw") is not None:
504 - video_grid_thw.extend(
505 + video_grid_thw.append(
506 mm_input["video_grid_thw"].tolist())
507 if mm_input.get("second_per_grid_ts") is not None:
507 - second_per_grid_ts.extend(
508 + second_per_grid_ts.append(
509 mm_input["second_per_grid_ts"])
510 if mm_input.get("audio_feature_lengths") is not None:
510 - audio_feature_lengths.extend(
511 + audio_feature_lengths.append(
512 mm_input["audio_feature_lengths"])
513 if mm_input.get("use_audio_in_video") is True:
514 use_audio_in_video = True
@@ -624,14 +625,23 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
625 ) -> BatchedTensorInputs:
626 if self.is_multimodal_raw_input_supported: # noqa: SIM102
627 if scheduler_output:
627 - multi_modal_kwargs_list = list[MultiModalKwargs]()
628 + mm_kwargs = list[MultiModalKwargsItem]()
629 for req in scheduler_output.scheduled_new_reqs:
629 - req_mm_inputs = req.mm_inputs
630 - if not isinstance(req_mm_inputs, list):
631 - req_mm_inputs = list(req_mm_inputs)
632 - multi_modal_kwargs_list.extend(req_mm_inputs)
630 + req_mm_kwargs = req.mm_kwargs
631 + if not isinstance(req_mm_kwargs, list):
632 + req_mm_kwargs = list(req_mm_kwargs)
633 + mm_kwargs.extend(req_mm_kwargs)
634 +
635 + # Input all modalities at once
636 + mm_kwargs_combined: BatchedTensorInputs = {}
637 + for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
638 + mm_kwargs,
639 + device=self.device,
640 + pin_memory=self.pin_memory,
641 + ):
642 + mm_kwargs_combined.update(mm_kwargs_group)
643
634 - return MultiModalKwargs.batch(multi_modal_kwargs_list)
644 + return mm_kwargs_combined
645
646 return {}
647
@@ -1146,13 +1156,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1156 return
1157
1158 # Batch the multi-modal inputs.
1149 - mm_inputs = list[MultiModalKwargs]()
1159 + mm_kwargs = list[MultiModalKwargsItem]()
1160 req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
1161 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
1162 req_state = self.requests[req_id]
1163
1164 for mm_input_id in encoder_input_ids:
1155 - mm_inputs.append(req_state.mm_inputs[mm_input_id])
1165 + mm_kwargs.append(req_state.mm_kwargs[mm_input_id])
1166 req_ids_pos.append(
1167 (req_id, mm_input_id, req_state.mm_positions[mm_input_id]))
1168
@@ -1163,17 +1173,12 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1173 # in the same batch while still being able to benefit from batching
1174 # multimodal inputs. The proper solution should be reordering the
1175 # encoder outputs.
1166 - grouped_mm_inputs_list = group_mm_inputs_by_modality(mm_inputs)
1167 -
1176 encoder_outputs = []
1169 - for grouped_mm_inputs in grouped_mm_inputs_list:
1170 - batched_mm_inputs = MultiModalKwargs.batch(
1171 - grouped_mm_inputs, pin_memory=self.pin_memory)
1172 - batched_mm_inputs = MultiModalKwargs.as_kwargs(
1173 - batched_mm_inputs,
1177 + for _, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
1178 + mm_kwargs,
1179 device=self.device,
1175 - )
1176 -
1180 + pin_memory=self.pin_memory,
1181 + ):
1182 # Run the encoder.
1183 # `curr_group_outputs` is either of the following:
1184 # 1. A tensor of shape (num_items, feature_size, hidden_size)
@@ -1182,11 +1187,11 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1187 # (feature_size, hidden_size) in case the feature size is dynamic
1188 # depending on the input multimodal items.
1189 curr_group_outputs = self.model.get_multimodal_embeddings(
1185 - **batched_mm_inputs)
1190 + **mm_kwargs_group)
1191
1192 sanity_check_mm_encoder_outputs(
1193 curr_group_outputs,
1189 - expected_num_items=len(grouped_mm_inputs),
1194 + expected_num_items=num_items,
1195 )
1196
1197 for output in curr_group_outputs:
@@ -1553,17 +1558,18 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1558
1559 input_ids = None
1560 inputs_embeds = self.inputs_embeds[:num_input_tokens]
1556 - model_mm_kwargs = self._extract_mm_kwargs(scheduler_output)
1557 - model_kwargs = self._init_model_kwargs(num_scheduled_tokens)
1561 + model_kwargs = {
1562 + **self._init_model_kwargs(num_scheduled_tokens),
1563 + **self._extract_mm_kwargs(scheduler_output),
1564 + }
1565 else:
1566 # For text-only models, we use token ids as input.
1567 # While it is possible to use embeddings as input just like the
1568 # multimodal models, it is not desirable for performance since
1569 # then the embedding layer is not included in the CUDA graph.
1570 input_ids = self.input_ids[:num_input_tokens]
1564 - model_kwargs = self._init_model_kwargs(num_input_tokens)
1571 inputs_embeds = None
1566 - model_mm_kwargs = {}
1572 + model_kwargs = self._init_model_kwargs(num_input_tokens)
1573 if self.uses_mrope:
1574 positions = self.mrope_positions[:, :num_input_tokens]
1575 else:
@@ -1596,10 +1602,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1602 positions=positions,
1603 intermediate_tensors=intermediate_tensors,
1604 inputs_embeds=inputs_embeds,
1599 - **MultiModalKwargs.as_kwargs(
1600 - model_mm_kwargs,
1601 - device=self.device,
1602 - ),
1605 **model_kwargs,
1606 )
1607
@@ -2196,14 +2198,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2198
2199 # Result in the maximum GPU consumption of the model
2200 dummy_mm_item = dummy_mm_data.get_item(modality=modality, item_index=0)
2199 - dummy_mm_kwargs = MultiModalKwargs.from_items([dummy_mm_item])
2201
2201 - batched_dummy_mm_inputs = MultiModalKwargs.batch([dummy_mm_kwargs] *
2202 - max_items_per_batch)
2203 - return MultiModalKwargs.as_kwargs(
2204 - batched_dummy_mm_inputs,
2205 - device=self.device,
2206 - )
2202 + return next(mm_kwargs_group
2203 + for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
2204 + [dummy_mm_item] * max_items_per_batch,
2205 + device=self.device,
2206 + pin_memory=self.pin_memory,
2207 + ))
2208
2209 @torch.inference_mode()
2210 def _dummy_run(
@@ -2269,15 +2270,17 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2270
2271 with self.maybe_dummy_run_with_lora(self.lora_config,
2272 num_scheduled_tokens):
2272 - model_kwargs = self._init_model_kwargs(num_tokens)
2273 if self.supports_mm_inputs:
2274 input_ids = None
2275 inputs_embeds = self.inputs_embeds[:num_tokens]
2276 - model_mm_kwargs = self._dummy_mm_kwargs(num_reqs)
2276 + model_kwargs = {
2277 + **self._init_model_kwargs(num_tokens),
2278 + **self._dummy_mm_kwargs(num_reqs),
2279 + }
2280 else:
2281 input_ids = self.input_ids[:num_tokens]
2282 inputs_embeds = None
2280 - model_mm_kwargs = {}
2283 + model_kwargs = self._init_model_kwargs(num_tokens)
2284
2285 if self.uses_mrope:
2286 positions = self.mrope_positions[:, :num_tokens]
@@ -2307,10 +2310,6 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2310 positions=positions,
2311 intermediate_tensors=intermediate_tensors,
2312 inputs_embeds=inputs_embeds,
2310 - **MultiModalKwargs.as_kwargs(
2311 - model_mm_kwargs,
2312 - device=self.device,
2313 - ),
2313 **model_kwargs,
2314 )
2315
vllm/v1/worker/tpu_model_runner.py
+17 -22
@@ -32,9 +32,9 @@ from vllm.model_executor.models.interfaces import supports_transcription
32 from vllm.model_executor.models.interfaces_base import (
33 is_pooling_model, is_text_generation_model)
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargs,
35 +from vllm.multimodal.inputs import (BatchedTensorInputs, MultiModalKwargsItem,
36 PlaceholderRange)
37 -from vllm.multimodal.utils import group_mm_inputs_by_modality
37 +from vllm.multimodal.utils import group_mm_kwargs_by_modality
38 from vllm.sequence import IntermediateTensors
39 from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
40 from vllm.utils import (LayerBlockType, cdiv, is_pin_memory_available,
@@ -394,7 +394,7 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
394 self.requests[req_id] = CachedRequestState(
395 req_id=req_id,
396 prompt_token_ids=new_req_data.prompt_token_ids,
397 - mm_inputs=new_req_data.mm_inputs,
397 + mm_kwargs=new_req_data.mm_kwargs,
398 mm_positions=new_req_data.mm_positions,
399 sampling_params=sampling_params,
400 pooling_params=None,
@@ -842,13 +842,13 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
842 return
843
844 # Batch the multi-modal inputs.
845 - mm_inputs = list[MultiModalKwargs]()
845 + mm_kwargs = list[MultiModalKwargsItem]()
846 req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
847 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
848 req_state = self.requests[req_id]
849
850 for mm_input_id in encoder_input_ids:
851 - mm_inputs.append(req_state.mm_inputs[mm_input_id])
851 + mm_kwargs.append(req_state.mm_kwargs[mm_input_id])
852 req_ids_pos.append(
853 (req_id, mm_input_id, req_state.mm_positions[mm_input_id]))
854
@@ -859,16 +859,12 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
859 # in the same batch while still being able to benefit from batching
860 # multimodal inputs. The proper solution should be reordering the
861 # encoder outputs.
862 - grouped_mm_inputs_list = group_mm_inputs_by_modality(mm_inputs)
863 -
862 encoder_outputs = []
865 - for grouped_mm_inputs in grouped_mm_inputs_list:
866 - batched_mm_inputs = MultiModalKwargs.batch(grouped_mm_inputs)
867 - batched_mm_inputs = MultiModalKwargs.as_kwargs(
868 - batched_mm_inputs,
863 + for _, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
864 + mm_kwargs,
865 device=self.device,
870 - )
871 -
866 + pin_memory=self.pin_memory,
867 + ):
868 # Run the encoder.
869 # `curr_group_outputs` is either of the following:
870 # 1. A tensor of shape (num_items, feature_size, hidden_size)
@@ -878,12 +874,12 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
874 # depending on the input multimodal items.
875 xm.mark_step()
876 curr_group_outputs = self.model.get_multimodal_embeddings(
881 - **batched_mm_inputs)
877 + **mm_kwargs_group)
878 xm.mark_step()
879
880 sanity_check_mm_encoder_outputs(
881 curr_group_outputs,
886 - expected_num_items=len(grouped_mm_inputs),
882 + expected_num_items=num_items,
883 )
884
885 if isinstance(curr_group_outputs, torch.Tensor):
@@ -1823,14 +1819,13 @@ class TPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
1819
1820 # Result in the maximum GPU consumption of the model
1821 dummy_mm_item = dummy_mm_data.get_item(modality=modality, item_index=0)
1826 - dummy_mm_kwargs = MultiModalKwargs.from_items([dummy_mm_item])
1822
1828 - batched_dummy_mm_inputs = MultiModalKwargs.batch([dummy_mm_kwargs] *
1829 - max_items_per_batch)
1830 - return MultiModalKwargs.as_kwargs(
1831 - batched_dummy_mm_inputs,
1832 - device=self.device,
1833 - )
1823 + return next(grouped_mm_kwargs
1824 + for _, _, grouped_mm_kwargs in group_mm_kwargs_by_modality(
1825 + [dummy_mm_item] * max_items_per_batch,
1826 + device=self.device,
1827 + pin_memory=self.pin_memory,
1828 + ))
1829
1830
1831 def _get_req_paddings(min_req_size: int, max_req_size: int) -> list[int]: