A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Refactor] Get prompt updates earlier (#23097)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 18, 2025 at 20:31 UTC
d3f71f1224403fef0d59ef73b894ac51800b8068
6 files changed
+84
-69
vllm/model_executor/models/deepseek_vl2.py
+3
-3
@@ -25,7 +25,8 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26
ImageSize, MultiModalDataItems)
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- BaseProcessingInfo, MultiModalHashes,
28
+ BaseProcessingInfo,
29
+ MultiModalProcessingInfo,
30
PromptReplacement, PromptUpdate)
31
from vllm.multimodal.profiling import BaseDummyInputsBuilder
32
from vllm.sequence import IntermediateTensors
@@ -291,8 +292,7 @@ class DeepseekVL2MultiModalProcessor(
292
tokenization_kwargs: Mapping[str, object],
293
*,
294
return_mm_hashes: bool,
294
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
295
- bool]:
295
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
296
# The processor logic is different for len(images) <= 2 vs > 2
297
# Since the processing cache assumes that the processor output is
298
# invariant of how many images are passed per prompt, we only
vllm/model_executor/models/h2ovl.py
+4
-4
@@ -20,8 +20,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.multimodal.inputs import MultiModalKwargsItems
21
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
22
MultiModalDataItems)
23
-from vllm.multimodal.processing import (MultiModalHashes, PromptReplacement,
24
- PromptUpdate, PromptUpdateDetails)
23
+from vllm.multimodal.processing import (MultiModalProcessingInfo,
24
+ PromptReplacement, PromptUpdate,
25
+ PromptUpdateDetails)
26
from vllm.transformers_utils.tokenizer import AnyTokenizer
27
28
from .intern_vit import InternVisionModel
@@ -480,8 +481,7 @@ class H2OVLMultiModalProcessor(
481
tokenization_kwargs: Mapping[str, object],
482
*,
483
return_mm_hashes: bool,
483
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
484
- bool]:
484
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
485
# The processor logic is different for len(images) <= 1 vs > 1
486
# Since the processing cache assumes that the processor output is
487
# invariant of how many images are passed per prompt, we only
vllm/model_executor/models/pixtral.py
+5
-10
@@ -39,7 +39,8 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
39
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
40
MultiModalDataItems)
41
from vllm.multimodal.processing import (BaseMultiModalProcessor,
42
- BaseProcessingInfo, MultiModalHashes,
42
+ BaseProcessingInfo,
43
+ MultiModalProcessingInfo,
44
PromptReplacement, PromptUpdate,
45
PromptUpdateDetails)
46
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -309,14 +310,8 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
310
tokenization_kwargs: Mapping[str, object],
311
*,
312
return_mm_hashes: bool,
312
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
313
- bool]:
314
- (
315
- prompt_ids,
316
- mm_kwargs,
317
- mm_hashes,
318
- _,
319
- ) = super()._cached_apply_hf_processor(
313
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
314
+ prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
315
prompt=prompt,
316
mm_data_items=mm_data_items,
317
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -325,7 +320,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
320
)
321
322
# NOTE: The tokens are already inserted by the chat template
328
- return prompt_ids, mm_kwargs, mm_hashes, True
323
+ return prompt_ids, mm_info, True
324
325
326
@MULTIMODAL_REGISTRY.register_processor(PixtralMultiModalProcessor,
vllm/model_executor/models/qwen2_5_omni_thinker.py
+15
-18
@@ -59,6 +59,7 @@ from vllm.multimodal.parse import (AudioProcessorItems, DictEmbeddingItems,
59
ModalityDataItems, MultiModalDataItems,
60
MultiModalDataParser)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
62
+ MultiModalPromptUpdates,
63
PlaceholderFeaturesInfo,
64
PromptReplacement, PromptUpdate)
65
from vllm.multimodal.profiling import BaseDummyInputsBuilder
@@ -88,10 +89,7 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
89
video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
90
video_grid_sizes = video_grid_thw.prod(-1)
91
91
- # vllm use `second_per_grid_ts` to compute multimodal rotary embedding
92
- video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
93
- if video_second_per_grid is not None:
94
- hf_inputs["second_per_grid_ts"] = video_second_per_grid
92
+ num_videos = len(video_grid_sizes)
93
94
return dict(
95
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
@@ -109,6 +107,7 @@ def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
107
"video", video_grid_sizes),
108
video_grid_thw=MultiModalFieldConfig.batched("video"),
109
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
110
+ use_audio_in_video=MultiModalFieldConfig.shared("video", num_videos),
111
)
112
113
@@ -251,6 +250,14 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
250
if ('audio_feature_lengths' not in hf_inputs
251
and feature_attention_mask is not None):
252
hf_inputs['audio_feature_lengths'] = feature_attention_mask.sum(-1)
253
+
254
+ video_second_per_grid = hf_inputs.get("video_second_per_grid", None)
255
+ if video_second_per_grid is not None:
256
+ hf_inputs["second_per_grid_ts"] = video_second_per_grid
257
+
258
+ use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
259
+ hf_inputs["use_audio_in_video"] = torch.tensor(use_audio_in_video)
260
+
261
return hf_inputs
262
263
def _get_mm_fields_config(
@@ -263,27 +270,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
270
def _maybe_apply_prompt_updates(
271
self,
272
mm_items: MultiModalDataItems,
266
- hf_processor_mm_kwargs: Mapping[str, object],
273
prompt_ids: list[int],
274
mm_kwargs: MultiModalKwargsItems,
275
+ mm_prompt_updates: MultiModalPromptUpdates,
276
is_update_applied: bool,
277
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
278
"""
279
Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
280
"""
274
- unbound_prompt_updates = self._get_prompt_updates(
275
- mm_items,
276
- hf_processor_mm_kwargs,
277
- mm_kwargs,
278
- )
279
- mm_prompt_updates = self._bind_and_group_updates(
280
- unbound_prompt_updates)
281
-
281
mm_item_counts = mm_items.get_all_counts()
282
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
283
285
- use_audio_in_video = hf_processor_mm_kwargs.get(
286
- "use_audio_in_video", False)
284
+ use_audio_in_video = (all(
285
+ item["use_audio_in_video"].data
286
+ for item in mm_kwargs["video"]) if "video" in mm_kwargs else False)
287
288
if is_update_applied:
289
mm_placeholders = self._find_mm_placeholders(
@@ -316,9 +316,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
316
tokenizer = self.info.get_tokenizer()
317
prompt = decode_tokens(tokenizer, prompt_ids)
318
319
- if use_audio_in_video:
320
- mm_kwargs["use_audio_in_video"] = True
321
-
319
return prompt_ids, prompt, mm_placeholders
320
321
def _get_prompt_updates(
vllm/model_executor/models/voxtral.py
+5
-6
@@ -35,7 +35,8 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
36
MultiModalDataParser)
37
from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
- BaseProcessingInfo, MultiModalHashes,
38
+ BaseProcessingInfo,
39
+ MultiModalProcessingInfo,
40
PromptReplacement, PromptUpdate)
41
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42
from vllm.sequence import IntermediateTensors
@@ -289,10 +290,8 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
290
tokenization_kwargs: Mapping[str, object],
291
*,
292
return_mm_hashes: bool,
292
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
293
- bool]:
294
- prompt_ids, mm_kwargs, mm_hashes, _ = super(
295
- )._cached_apply_hf_processor(
293
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
294
+ prompt_ids, mm_info, _ = super()._cached_apply_hf_processor(
295
prompt=prompt,
296
mm_data_items=mm_data_items,
297
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -301,7 +300,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]
300
)
301
302
# NOTE: The tokens are already inserted by the chat template
304
- return prompt_ids, mm_kwargs, mm_hashes, True
303
+ return prompt_ids, mm_info, True
304
305
def _get_data_parser(self) -> MultiModalDataParser:
306
sampling_rate = self.info.get_hf_processor().sampling_rate
vllm/multimodal/processing.py
+52
-28
@@ -989,6 +989,18 @@ A collection of hashes with a similar structure as
989
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
990
"""
991
992
+MultiModalPromptUpdates = dict[str, Sequence[BoundPromptUpdate]]
993
+"""
994
+A collection of prompt updates with a similar structure as
995
+[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
996
+"""
997
+
998
+
999
+class MultiModalProcessingInfo(NamedTuple):
1000
+ kwargs: MultiModalKwargsItems
1001
+ hashes: Optional[MultiModalHashes]
1002
+ prompt_updates: MultiModalPromptUpdates
1003
+
1004
1005
class BaseMultiModalProcessor(ABC, Generic[_I]):
1006
"""
@@ -1363,7 +1375,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1375
cache: ProcessingCache,
1376
mm_cache_items_or_hashes: dict[str, list[_CacheItemOrHash]],
1377
mm_missing_kwargs: MultiModalKwargsItems,
1366
- ) -> dict[str, list[MultiModalKwargsItem]]:
1378
+ ) -> MultiModalKwargsItems:
1379
mm_missing_next_idx = defaultdict[str, int](lambda: 0)
1380
1381
merged_items = defaultdict[str, list[MultiModalKwargsItem]](list)
@@ -1379,7 +1391,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1391
1392
merged_items[modality].append(kw_item)
1393
1382
- return dict(merged_items)
1394
+ return MultiModalKwargsItems(merged_items)
1395
1396
def _apply_hf_processor(
1397
self,
@@ -1389,8 +1401,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1401
tokenization_kwargs: Mapping[str, object],
1402
*,
1403
return_mm_hashes: bool,
1392
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
1393
- bool]:
1404
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1405
(
1406
prompt_ids,
1407
mm_processed_data,
@@ -1413,7 +1424,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1424
tokenization_kwargs)
1425
if return_mm_hashes else None)
1426
1416
- return prompt_ids, mm_kwargs, mm_hashes, is_update_applied
1427
+ unbound_prompt_updates = self._get_prompt_updates(
1428
+ mm_data_items,
1429
+ hf_processor_mm_kwargs,
1430
+ mm_kwargs,
1431
+ )
1432
+ mm_prompt_updates = self._bind_and_group_updates(
1433
+ unbound_prompt_updates)
1434
+
1435
+ mm_info = MultiModalProcessingInfo(
1436
+ kwargs=mm_kwargs,
1437
+ hashes=mm_hashes,
1438
+ prompt_updates=mm_prompt_updates,
1439
+ )
1440
+
1441
+ return prompt_ids, mm_info, is_update_applied
1442
1443
def _cached_apply_hf_processor(
1444
self,
@@ -1423,8 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1448
tokenization_kwargs: Mapping[str, object],
1449
*,
1450
return_mm_hashes: bool,
1426
- ) -> tuple[list[int], MultiModalKwargsItems, Optional[MultiModalHashes],
1427
- bool]:
1451
+ ) -> tuple[list[int], MultiModalProcessingInfo, bool]:
1452
"""
1453
Apply the HF processor on the full prompt text,
1454
caching the results and reusing cached results.
@@ -1475,18 +1499,27 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1499
hf_processor_mm_kwargs),
1500
)
1501
1478
- mm_cache_items_merged = self._merge_mm_kwargs(
1502
+ mm_kwargs = self._merge_mm_kwargs(
1503
cache,
1504
mm_cache_items_or_hashes=mm_cache_items_or_hashes,
1505
mm_missing_kwargs=mm_missing_kwargs,
1506
)
1507
1484
- mm_kwargs = MultiModalKwargsItems.from_seq([
1485
- item for cache_items in mm_cache_items_merged.values()
1486
- for item in cache_items
1487
- ])
1508
+ unbound_prompt_updates = self._get_prompt_updates(
1509
+ mm_data_items,
1510
+ hf_processor_mm_kwargs,
1511
+ mm_kwargs,
1512
+ )
1513
+ mm_prompt_updates = self._bind_and_group_updates(
1514
+ unbound_prompt_updates)
1515
+
1516
+ mm_info = MultiModalProcessingInfo(
1517
+ kwargs=mm_kwargs,
1518
+ hashes=mm_hashes_to_return,
1519
+ prompt_updates=mm_prompt_updates,
1520
+ )
1521
1489
- return prompt_ids, mm_kwargs, mm_hashes_to_return, is_update_applied
1522
+ return prompt_ids, mm_info, is_update_applied
1523
1524
def _bind_and_group_updates(
1525
self,
@@ -1626,19 +1659,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1659
def _maybe_apply_prompt_updates(
1660
self,
1661
mm_items: MultiModalDataItems,
1629
- hf_processor_mm_kwargs: Mapping[str, object],
1662
prompt_ids: list[int],
1663
mm_kwargs: MultiModalKwargsItems,
1664
+ mm_prompt_updates: MultiModalPromptUpdates,
1665
is_update_applied: bool,
1666
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1634
- unbound_prompt_updates = self._get_prompt_updates(
1635
- mm_items,
1636
- hf_processor_mm_kwargs,
1637
- mm_kwargs,
1638
- )
1639
- mm_prompt_updates = self._bind_and_group_updates(
1640
- unbound_prompt_updates)
1641
-
1667
mm_item_counts = mm_items.get_all_counts()
1668
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1669
@@ -1694,8 +1719,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1719
1720
(
1721
prompt_ids,
1697
- mm_kwargs,
1698
- mm_hashes,
1722
+ mm_info,
1723
is_update_applied,
1724
) = self._cached_apply_hf_processor(
1725
prompt,
@@ -1708,9 +1732,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1732
# NOTE: tokenization_kwargs are not required to init processor
1733
prompt_ids, prompt, mm_placeholders = self._maybe_apply_prompt_updates(
1734
mm_items=mm_items,
1711
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1735
prompt_ids=prompt_ids,
1713
- mm_kwargs=mm_kwargs,
1736
+ mm_kwargs=mm_info.kwargs,
1737
+ mm_prompt_updates=mm_info.prompt_updates,
1738
is_update_applied=is_update_applied,
1739
)
1740
@@ -1723,8 +1747,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1747
type="multimodal",
1748
prompt=prompt,
1749
prompt_token_ids=prompt_ids,
1726
- mm_kwargs=mm_kwargs,
1727
- mm_hashes=mm_hashes,
1750
+ mm_kwargs=mm_info.kwargs,
1751
+ mm_hashes=mm_info.hashes,
1752
mm_placeholders=mm_placeholder_ranges,
1753
)
1754