A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove `MultiModalPlaceholderMap` (#25366)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 22, 2025 at 16:49 UTC
f92d952632541889c38f107f0425a5bd4707d9d9
7 files changed
+2
-128
tests/kernels/utils.py
-2
@@ -959,7 +959,6 @@ def make_test_metadata(
959
return attn_backend_obj.make_metadata(
960
num_prefills=num_prefills,
961
slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
962
- multi_modal_placeholder_index_maps=None,
962
enable_kv_scales_calculation=True,
963
num_prefill_tokens=num_prefill_tokens,
964
num_decode_tokens=num_decode_tokens,
@@ -1009,7 +1008,6 @@ def make_test_metadata(
1008
return attn_backend_obj.make_metadata(
1009
num_prefills=num_prefills,
1010
slot_mapping=kv_mmap.slot_mapping,
1012
- multi_modal_placeholder_index_maps=None,
1011
enable_kv_scales_calculation=True,
1012
num_prefill_tokens=num_prefill_tokens,
1013
num_decode_tokens=num_decode_tokens,
vllm/attention/backends/abstract.py
-10
@@ -10,7 +10,6 @@ from typing import (Any, Dict, Generic, List, Optional, Protocol, Set, Tuple,
10
import torch
11
12
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
13
-from vllm.multimodal import MultiModalPlaceholderMap
13
14
15
class AttentionType:
@@ -116,15 +115,6 @@ class AttentionMetadata:
115
# in block 0, and 1st slot in block 1, respectively.
116
slot_mapping: torch.Tensor
117
119
- # The index maps that relate multi-modal embeddings to the corresponding
120
- # placeholders.
121
- #
122
- # N.B. These aren't really related to attention and don't belong on this
123
- # type -- this is just a temporary solution to make them available to
124
- # `model_executable`.
125
- multi_modal_placeholder_index_maps: Optional[Dict[
126
- str, MultiModalPlaceholderMap.IndexMap]]
127
-
118
# Enable/disable KV scales calculation. This is so that we can disable the
119
# calculation until after prefill and cuda graph capture.
120
enable_kv_scales_calculation: bool
vllm/attention/backends/placeholder_attn.py
+1
-22
@@ -1,10 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections import defaultdict
4
from dataclasses import dataclass
5
from itertools import accumulate
7
-from typing import Dict, List, Optional, Tuple, Type
6
+from typing import List, Optional, Tuple, Type
7
8
import torch
9
@@ -12,7 +11,6 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
AttentionMetadata,
12
AttentionMetadataBuilder)
13
from vllm.attention.backends.utils import CommonAttentionState
15
-from vllm.multimodal import MultiModalPlaceholderMap
14
from vllm.utils import async_tensor_h2d
15
16
# Placeholder attention backend for models like Mamba and pooling models that
@@ -141,8 +139,6 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
139
num_prefill_tokens=self.num_prefill_tokens,
140
num_decode_tokens=0,
141
slot_mapping=slot_mapping,
144
- multi_modal_placeholder_index_maps=self.
145
- multi_modal_placeholder_index_maps,
142
enable_kv_scales_calculation=self.enable_kv_scales_calculation,
143
seq_lens=seq_lens,
144
seq_lens_tensor=seq_lens_tensor,
@@ -178,7 +174,6 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
174
num_prefill_tokens=0,
175
num_decode_tokens=self.num_decode_tokens,
176
slot_mapping=slot_mapping,
181
- multi_modal_placeholder_index_maps=None,
177
enable_kv_scales_calculation=True,
178
seq_lens=None,
179
seq_lens_tensor=seq_lens_tensor,
@@ -210,9 +205,6 @@ class PlaceholderAttentionMetadataBuilder(
205
self.prefill_seq_lens: List[int] = []
206
self.context_lens: List[int] = []
207
self.curr_seq_lens: List[int] = []
213
- self.multimodal_placeholder_maps: Dict[
214
- str,
215
- MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
208
self.num_prefills = 0
209
self.num_prefill_tokens = 0
210
self.num_decode_tokens = 0
@@ -232,12 +224,6 @@ class PlaceholderAttentionMetadataBuilder(
224
self.context_lens.append(context_len)
225
226
if is_prompt:
235
- mm_maps = inter_data.multi_modal_placeholder_maps
236
- if mm_maps:
237
- for modality, placeholders in mm_maps.items():
238
- self.multimodal_placeholder_maps[modality].extend(
239
- placeholders)
240
-
227
self.num_prefills += 1
228
self.num_prefill_tokens += token_len
229
self.prefill_seq_lens.append(seq_len)
@@ -295,12 +281,6 @@ class PlaceholderAttentionMetadataBuilder(
281
seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
282
device, self.runner.pin_memory)
283
298
- placeholder_index_maps = {
299
- modality: placeholder_map.index_map()
300
- for modality, placeholder_map in
301
- self.multimodal_placeholder_maps.items()
302
- }
303
-
284
# Placeholders
285
slot_mapping_tensor = torch.empty(0)
286
block_tables = torch.empty(0)
@@ -308,7 +288,6 @@ class PlaceholderAttentionMetadataBuilder(
288
return PlaceholderAttentionMetadata(
289
num_prefills=self.num_prefills,
290
slot_mapping=slot_mapping_tensor,
311
- multi_modal_placeholder_index_maps=placeholder_index_maps,
291
enable_kv_scales_calculation=True,
292
num_prefill_tokens=self.num_prefill_tokens,
293
num_decode_tokens=num_decode_tokens,
vllm/attention/backends/utils.py
-18
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Attention backend utils"""
4
-from collections import defaultdict
4
from contextlib import contextmanager
5
from dataclasses import dataclass
6
from itertools import accumulate
@@ -15,7 +14,6 @@ from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
14
from vllm.attention.backends.abstract import AttentionType
15
from vllm.config import ModelConfig
16
from vllm.logger import init_logger
18
-from vllm.multimodal import MultiModalPlaceholderMap
17
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
18
19
logger = init_logger(__name__)
@@ -135,9 +133,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
133
self.context_lens: List[int] = []
134
self.block_tables: List[List[int]] = []
135
self.curr_seq_lens: List[int] = []
138
- self.multimodal_placeholder_maps: Dict[
139
- str,
140
- MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
136
self.num_prefills = 0
137
self.num_prefill_tokens = 0
138
self.num_decode_tokens = 0
@@ -154,12 +149,6 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
149
inter_data.curr_sliding_window_blocks):
150
self.context_lens.append(context_len)
151
if is_prompt:
157
- mm_maps = inter_data.multi_modal_placeholder_maps
158
- if mm_maps:
159
- for modality, placeholders in mm_maps.items():
160
- self.multimodal_placeholder_maps[modality].extend(
161
- placeholders)
162
-
152
self.num_prefills += 1
153
self.num_prefill_tokens += token_len
154
self.prefill_seq_lens.append(seq_len)
@@ -254,16 +243,10 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
243
self.runner.pin_memory)
244
seq_start_loc_tensor = async_tensor_h2d(seq_start_loc, torch.int32,
245
device, self.runner.pin_memory)
257
- placeholder_index_maps = {
258
- modality: placeholder_map.index_map()
259
- for modality, placeholder_map in
260
- self.multimodal_placeholder_maps.items()
261
- }
246
247
return self._metadata_cls( # type: ignore
248
num_prefills=self.num_prefills,
249
slot_mapping=slot_mapping_tensor,
266
- multi_modal_placeholder_index_maps=placeholder_index_maps,
250
enable_kv_scales_calculation=True,
251
num_prefill_tokens=self.num_prefill_tokens,
252
num_decode_tokens=num_decode_tokens,
@@ -320,7 +303,6 @@ class CommonAttentionState(AttentionState):
303
num_prefill_tokens=0,
304
num_decode_tokens=batch_size,
305
slot_mapping=self._graph_slot_mapping[:batch_size],
323
- multi_modal_placeholder_index_maps=None,
306
enable_kv_scales_calculation=True,
307
seq_lens=None,
308
seq_lens_tensor=self._graph_seq_lens[:batch_size],
vllm/multimodal/__init__.py
-2
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from .base import MultiModalPlaceholderMap
3
from .hasher import MultiModalHasher
4
from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
5
MultiModalDataDict, MultiModalKwargs,
@@ -27,7 +26,6 @@ __all__ = [
26
"MultiModalKwargs",
27
"MultiModalKwargsItems",
28
"MultiModalPlaceholderDict",
30
- "MultiModalPlaceholderMap",
29
"MultiModalUUIDDict",
30
"NestedTensors",
31
"MULTIMODAL_REGISTRY",
vllm/multimodal/base.py
+1
-73
@@ -3,83 +3,11 @@
3
4
from abc import ABC, abstractmethod
5
from pathlib import Path
6
-from typing import Generic, NamedTuple, TypeVar
6
+from typing import Generic, TypeVar
7
8
_T = TypeVar("_T")
9
10
11
-class MultiModalPlaceholderMap:
12
- """
13
- Relates multi-modal embeddings to their corresponding placeholders.
14
-
15
- Note: This is only used in V0.
16
- """
17
-
18
- class IndexMap(NamedTuple):
19
- src: list[int]
20
- dest: list[int]
21
-
22
- src_ranges: list[range]
23
- """
24
- The indices of the multi-modal embeddings that will replace the
25
- corresponding placeholder embeddings pointed to by ``dest_ranges``.
26
- """
27
-
28
- src_len: int
29
- """
30
- The total number of flattened multi-modal embeddings.
31
- """
32
-
33
- dest_ranges: list[range]
34
- """
35
- The indices of the placeholder embeddings that will be replaced by the
36
- multimodal embeddings.
37
- """
38
-
39
- dest_len: int
40
- """
41
- The total number of embeddings in the destination tensor.
42
- """
43
-
44
- def __init__(self):
45
- self.src_ranges = []
46
- self.src_len = 0
47
- self.dest_ranges = []
48
- self.dest_len = 0
49
-
50
- def extend(self, other: "MultiModalPlaceholderMap"):
51
- """
52
- Adds the placeholders from another ``MultiModalPlaceholderMap`` to this
53
- instance based on the source and destination tensors being
54
- concatenated.
55
- """
56
-
57
- self.src_ranges.extend(
58
- range(self.src_len + r.start, self.src_len + r.stop)
59
- for r in other.src_ranges)
60
- self.src_len += other.src_len
61
- self.dest_ranges.extend(
62
- range(self.dest_len + r.start, self.dest_len + r.stop)
63
- for r in other.dest_ranges)
64
- self.dest_len += other.dest_len
65
-
66
- def index_map(self) -> "IndexMap":
67
- """
68
- Finalizes the placeholder map into lists of indices that can be used to
69
- index the source and destination tensors.
70
- """
71
-
72
- src_indices = [i for r in self.src_ranges for i in r]
73
- dest_indices = [i for r in self.dest_ranges for i in r]
74
-
75
- if len(src_indices) != len(dest_indices):
76
- raise ValueError(
77
- f"The number of source ({len(src_indices)}) and destination "
78
- f"indices ({len(dest_indices)}) must be the same.")
79
-
80
- return self.IndexMap(src=src_indices, dest=dest_indices)
81
-
82
-
11
class MediaIO(ABC, Generic[_T]):
12
13
@abstractmethod
vllm/v1/attention/backends/cpu_attn.py
-1
@@ -425,7 +425,6 @@ class TorchSDPAMetadataBuilderV1(AttentionMetadataBuilder[TorchSDPAMetadata]):
425
num_prompt_req], # prefill
426
query_start_loc=query_start_loc_cpu[:num_reqs +
427
1], # for logits index
428
- multi_modal_placeholder_index_maps=None,
428
enable_kv_scales_calculation=False,
429
)
430