A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Fix backward compatibility from #23030 (#23070)
Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: Roger Wang <hey@rogerw.me>
Roger Wang committed
Aug 17, 2025 at 23:33 UTC
89657a557c6831cca9fa5e59822af0cf27d67a98
3 files changed
+12
-7
vllm/multimodal/base.py
+6
-3
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Generic, NamedTuple, TypeVar
9
if TYPE_CHECKING:
10
from vllm.sequence import SequenceGroupMetadata
11
12
-from .inputs import MultiModalKwargs, PlaceholderRange
12
+from .inputs import MultiModalKwargs, NestedTensors, PlaceholderRange
13
14
_T = TypeVar("_T")
15
@@ -56,7 +56,8 @@ class MultiModalPlaceholderMap:
56
@classmethod
57
def from_seq_group(
58
cls, seq_group: "SequenceGroupMetadata", positions: range
59
- ) -> tuple[MultiModalKwargs, dict[str, "MultiModalPlaceholderMap"]]:
59
+ ) -> tuple[dict[str, NestedTensors], dict[str,
60
+ "MultiModalPlaceholderMap"]]:
61
"""
62
Returns the multi-modal items that intersect with the portion of a
63
prompt (``seq_group``) represented by ``positions``, as well as a
@@ -99,7 +100,7 @@ class MultiModalPlaceholderMap:
100
seq_mm_placeholders = seq_group.multi_modal_placeholders
101
102
if not seq_mm_data or not seq_mm_placeholders:
102
- return MultiModalKwargs(), {}
103
+ return MultiModalKwargs().get_data(), {}
104
105
placeholder_maps = dict[str, MultiModalPlaceholderMap]()
106
@@ -116,6 +117,8 @@ class MultiModalPlaceholderMap:
117
118
placeholder_maps[modality] = placeholder_map
119
120
+ seq_mm_data = seq_mm_data if isinstance(
121
+ seq_mm_data, dict) else seq_mm_data.get_data()
122
return seq_mm_data, placeholder_maps
123
124
def append_items_from_seq_group(
vllm/multimodal/inputs.py
+3
-3
@@ -664,7 +664,7 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
664
def modality(self) -> str:
665
return self._modality
666
667
- def get_data(self) -> Mapping[str, NestedTensors]:
667
+ def get_data(self) -> dict[str, NestedTensors]:
668
return {key: elem.data for key, elem in self.items()}
669
670
@@ -720,7 +720,7 @@ class MultiModalKwargs:
720
items_by_modality = full_groupby(items, key=lambda x: x.modality)
721
self._items_by_modality = dict(items_by_modality)
722
723
- self._data: Optional[Mapping[str, NestedTensors]] = None
723
+ self._data: Optional[dict[str, NestedTensors]] = None
724
725
@property
726
def modalities(self):
@@ -883,7 +883,7 @@ class MultiModalKwargs:
883
884
def get_data(self,
885
*,
886
- pin_memory: bool = False) -> Mapping[str, NestedTensors]:
886
+ pin_memory: bool = False) -> dict[str, NestedTensors]:
887
if self._data is not None:
888
return self._data
889
vllm/sequence.py
+3
-1
@@ -22,6 +22,7 @@ from vllm.pooling_params import PoolingParams
22
from vllm.sampling_params import RequestOutputKind, SamplingParams
23
24
if TYPE_CHECKING:
25
+ from vllm.multimodal.inputs import NestedTensors
26
from vllm.v1.worker.kv_connector_model_runner_mixin import (
27
KVConnectorOutput)
28
@@ -978,7 +979,8 @@ class SequenceGroupMetadata(
979
state: Optional[SequenceGroupState] = msgspec.field(
980
default_factory=lambda: SequenceGroupState())
981
token_type_ids: Optional[list[int]] = None
981
- multi_modal_data: Optional[MultiModalKwargs] = None
982
+ multi_modal_data: Optional[Union[MultiModalKwargs,
983
+ dict[str, "NestedTensors"]]] = None
984
multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
985
encoder_seq_data: Optional[SequenceData] = None
986
cross_block_table: Optional[list[int]] = None