A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 04:00 UTC
[Misc] Rename `group_mm_kwargs_by_modality -> group_and_batch_mm_kwargs` (#36158)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 6, 2026 at 12:32 UTC
6dd302653f82148ad44d9766fdc3daede0ede040
4 files changed
+37
-28
tests/models/multimodal/processing/test_tensor_schema.py
+2
-2
@@ -27,7 +27,7 @@ from vllm.distributed import (
27
from vllm.model_executor.models.interfaces import supports_multimodal
28
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
29
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
30
-from vllm.multimodal.utils import group_mm_kwargs_by_modality
30
+from vllm.multimodal.utils import group_and_batch_mm_kwargs
31
from vllm.platforms import current_platform
32
from vllm.tokenizers import cached_tokenizer_from_config
33
from vllm.utils.collection_utils import is_list_of
@@ -114,7 +114,7 @@ def create_batched_mm_kwargs(
114
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
115
)["mm_kwargs"].require_data()
116
117
- return group_mm_kwargs_by_modality(
117
+ return group_and_batch_mm_kwargs(
118
[
119
(modality, item)
120
for modality in supported_mm_limits
vllm/multimodal/utils.py
+15
-1
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any
10
import numpy as np
11
import numpy.typing as npt
12
from PIL import Image
13
+from typing_extensions import deprecated
14
15
from vllm.utils.import_utils import LazyLoader
16
@@ -207,7 +208,7 @@ def group_and_batch_mm_items(
208
assert start_idx == len(items)
209
210
210
-def group_mm_kwargs_by_modality(
211
+def group_and_batch_mm_kwargs(
212
mm_kwargs: list[tuple[str, MultiModalKwargsItem]],
213
*,
214
device: torch.types.Device = None,
@@ -246,6 +247,19 @@ def group_mm_kwargs_by_modality(
247
yield modality, num_items, mm_kwargs_batch
248
249
250
+@deprecated(
251
+ "`group_mm_kwargs_by_modality` has been renamed to `group_and_batch_mm_kwargs`. "
252
+ "The old name will be removed in v0.19."
253
+)
254
+def group_mm_kwargs_by_modality(
255
+ mm_kwargs: list[tuple[str, MultiModalKwargsItem]],
256
+ *,
257
+ device: torch.types.Device = None,
258
+ pin_memory: bool = False,
259
+) -> Generator[tuple[str, int, BatchedTensorInputs], None, None]:
260
+ return group_and_batch_mm_kwargs(mm_kwargs, device=device, pin_memory=pin_memory)
261
+
262
+
263
def fetch_audio(
264
audio_url: str,
265
audio_io_kwargs: dict[str, Any] | None = None,
vllm/v1/worker/gpu/mm/encoder_runner.py
+5
-7
@@ -5,7 +5,7 @@ import torch
5
6
from vllm.model_executor.models.interfaces import SupportsMultiModal
7
from vllm.multimodal.inputs import MultiModalKwargsItem
8
-from vllm.multimodal.utils import group_mm_kwargs_by_modality
8
+from vllm.multimodal.utils import group_and_batch_mm_kwargs
9
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
10
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
11
@@ -53,14 +53,12 @@ class EncoderRunner:
53
mm_kwargs: list[tuple[str, MultiModalKwargsItem]],
54
) -> list[torch.Tensor]:
55
encoder_outputs: list[torch.Tensor] = []
56
- for modality, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
56
+ for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
57
mm_kwargs, device=self.device, pin_memory=False
58
):
59
- curr_group_outputs = self.model.embed_multimodal(**mm_kwargs_group)
60
- sanity_check_mm_encoder_outputs(
61
- curr_group_outputs, expected_num_items=num_items
62
- )
63
- encoder_outputs.extend(curr_group_outputs)
59
+ batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
60
+ sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
61
+ encoder_outputs.extend(batch_outputs)
62
return encoder_outputs
63
64
def gather_mm_embeddings(
vllm/v1/worker/gpu_model_runner.py
+15
-18
@@ -93,7 +93,7 @@ from vllm.multimodal.inputs import (
93
MultiModalKwargsItem,
94
PlaceholderRange,
95
)
96
-from vllm.multimodal.utils import group_mm_kwargs_by_modality
96
+from vllm.multimodal.utils import group_and_batch_mm_kwargs
97
from vllm.pooling_params import PoolingParams
98
from vllm.sampling_params import SamplingType
99
from vllm.sequence import IntermediateTensors
@@ -1311,12 +1311,12 @@ class GPUModelRunner(
1311
1312
# Input all modalities at once
1313
mm_kwargs_combined: BatchedTensorInputs = {}
1314
- for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
1314
+ for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
1315
mm_kwargs,
1316
device=self.device,
1317
pin_memory=self.pin_memory,
1318
):
1319
- mm_kwargs_combined.update(mm_kwargs_group)
1319
+ mm_kwargs_combined.update(mm_kwargs_batch)
1320
1321
return mm_kwargs_combined
1322
@@ -2446,12 +2446,12 @@ class GPUModelRunner(
2446
encoder_outputs: list[torch.Tensor] = []
2447
# Track the current index in mm_kwargs/mm_lora_refs to map groups to request IDs
2448
current_item_idx = 0
2449
- for modality, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
2449
+ for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
2450
mm_kwargs,
2451
device=self.device,
2452
pin_memory=self.pin_memory,
2453
):
2454
- curr_group_outputs: MultiModalEmbeddings
2454
+ batch_outputs: MultiModalEmbeddings
2455
2456
# EVS-related change.
2457
# (ekhvedchenia): Temporary hack to limit peak memory usage when
@@ -2467,14 +2467,14 @@ class GPUModelRunner(
2467
and modality == "video"
2468
and num_items > 1
2469
):
2470
- curr_group_outputs_lst = list[torch.Tensor]()
2470
+ batch_outputs_lst = list[torch.Tensor]()
2471
for video_idx in range(num_items):
2472
video_mm_kwargs_item = mm_kwargs[current_item_idx + video_idx]
2473
with self.timed_encoder_operation(
2474
should_time, mm_lora_refs, current_item_idx + video_idx, 1
2475
):
2476
_, _, micro_batch_mm_inputs = next(
2477
- group_mm_kwargs_by_modality(
2477
+ group_and_batch_mm_kwargs(
2478
[video_mm_kwargs_item],
2479
device=self.device,
2480
pin_memory=self.pin_memory,
@@ -2485,12 +2485,12 @@ class GPUModelRunner(
2485
**micro_batch_mm_inputs
2486
)
2487
2488
- curr_group_outputs_lst.extend(micro_batch_outputs)
2488
+ batch_outputs_lst.extend(micro_batch_outputs)
2489
2490
- curr_group_outputs = curr_group_outputs_lst
2490
+ batch_outputs = batch_outputs_lst
2491
else:
2492
# Run the encoder.
2493
- # `curr_group_outputs` is either of the following:
2493
+ # `batch_outputs` is either of the following:
2494
# 1. A tensor of shape (num_items, feature_size, hidden_size)
2495
# in case feature_size is fixed across all multimodal items.
2496
# 2. A list or tuple (length: num_items) of tensors,
@@ -2500,13 +2500,10 @@ class GPUModelRunner(
2500
with self.timed_encoder_operation(
2501
should_time, mm_lora_refs, current_item_idx, num_items
2502
):
2503
- curr_group_outputs = model.embed_multimodal(**mm_kwargs_group)
2503
+ batch_outputs = model.embed_multimodal(**mm_kwargs_batch)
2504
2505
- sanity_check_mm_encoder_outputs(
2506
- curr_group_outputs,
2507
- expected_num_items=num_items,
2508
- )
2509
- encoder_outputs.extend(curr_group_outputs)
2505
+ sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
2506
+ encoder_outputs.extend(batch_outputs)
2507
2508
current_item_idx += num_items
2509
@@ -4707,8 +4704,8 @@ class GPUModelRunner(
4704
assert dummy_mm_item is not None, "Item should not already be cached"
4705
4706
return next(
4710
- mm_kwargs_group
4711
- for _, _, mm_kwargs_group in group_mm_kwargs_by_modality(
4707
+ mm_kwargs_batch
4708
+ for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
4709
[(modality, dummy_mm_item)] * max_items_per_batch,
4710
device=self.device,
4711
pin_memory=self.pin_memory,