A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Core][VLM] Add precise multi-modal placeholder tracking (#8346)
Signed-off-by: Peter Salas <peter@fixie.ai>
Peter Salas committed
Nov 1, 2024 at 16:21 UTC
6c0b7f548d80b5f61bfa472ad1497597c922dbc2
53 files changed
+914
-282
examples/offline_inference_audio_language.py
+1
-5
@@ -34,11 +34,7 @@ def run_ultravox(question: str, audio_count: int):
34
tokenize=False,
35
add_generation_prompt=True)
36
37
- llm = LLM(model=model_name,
38
- enforce_eager=True,
39
- enable_chunked_prefill=False,
40
- max_model_len=8192,
41
- limit_mm_per_prompt={"audio": audio_count})
37
+ llm = LLM(model=model_name, limit_mm_per_prompt={"audio": audio_count})
38
stop_token_ids = None
39
return llm, prompt, stop_token_ids
40
tests/kernels/utils.py
+2
@@ -869,6 +869,7 @@ def make_test_metadata(
869
return attn_backend.make_metadata(
870
num_prefills=num_prefills,
871
slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
872
+ multi_modal_placeholder_index_maps=None,
873
num_prefill_tokens=num_prefill_tokens,
874
num_decode_tokens=num_decode_tokens,
875
seq_lens=seq_lens,
@@ -914,6 +915,7 @@ def make_test_metadata(
915
return attn_backend.make_metadata(
916
num_prefills=num_prefills,
917
slot_mapping=kv_mmap.slot_mapping,
918
+ multi_modal_placeholder_index_maps=None,
919
num_prefill_tokens=num_prefill_tokens,
920
num_decode_tokens=num_decode_tokens,
921
seq_lens=seq_lens,
tests/models/decoder_only/audio_language/test_ultravox.py
+74
-17
@@ -2,8 +2,10 @@ from typing import List, Optional, Tuple, Type
2
3
import numpy as np
4
import pytest
5
+import pytest_asyncio
6
from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8
+from tests.utils import RemoteOpenAIServer
9
from vllm.sequence import SampleLogprobs
10
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
@@ -17,6 +19,13 @@ AudioTuple = Tuple[np.ndarray, int]
19
VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
20
HF_PLACEHOLDER = "<|audio|>"
21
22
+CHUNKED_PREFILL_KWARGS = {
23
+ "enable_chunked_prefill": True,
24
+ "max_num_seqs": 2,
25
+ # Use a very small limit to exercise chunked prefill.
26
+ "max_num_batched_tokens": 16
27
+}
28
+
29
30
@pytest.fixture(scope="session")
31
def audio_assets():
@@ -30,6 +39,26 @@ def audio(request):
39
return AudioAsset(request.param)
40
41
42
+@pytest.fixture(params=({}, CHUNKED_PREFILL_KWARGS))
43
+def server(request, audio_assets):
44
+ args = [
45
+ "--dtype=bfloat16", "--max-model-len=4096", "--enforce-eager",
46
+ f"--limit-mm-per-prompt=audio={len(audio_assets)}"
47
+ ] + [
48
+ f"--{key.replace('_','-')}={value}"
49
+ for key, value in request.param.items()
50
+ ]
51
+
52
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
53
+ yield remote_server
54
+
55
+
56
+@pytest_asyncio.fixture
57
+async def client(server):
58
+ async with server.get_async_client() as async_client:
59
+ yield async_client
60
+
61
+
62
def _get_prompt(audio_count, question, placeholder):
63
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
64
placeholder = f"{placeholder}\n" * audio_count
@@ -68,8 +97,7 @@ def run_test(
97
dtype: str,
98
max_tokens: int,
99
num_logprobs: int,
71
- tensor_parallel_size: int,
72
- distributed_executor_backend: Optional[str] = None,
100
+ **kwargs,
101
):
102
"""Inference result should be the same between hf and vllm."""
103
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
@@ -79,11 +107,8 @@ def run_test(
107
# if we run HF first, the cuda initialization will be done and it
108
# will hurt multiprocessing backend with fork method (the default method).
109
82
- with vllm_runner(model,
83
- dtype=dtype,
84
- tensor_parallel_size=tensor_parallel_size,
85
- distributed_executor_backend=distributed_executor_backend,
86
- enforce_eager=True) as vllm_model:
110
+ with vllm_runner(model, dtype=dtype, enforce_eager=True,
111
+ **kwargs) as vllm_model:
112
vllm_outputs_per_audio = [
113
vllm_model.generate_greedy_logprobs([vllm_prompt],
114
max_tokens,
@@ -135,18 +160,16 @@ def run_multi_audio_test(
160
dtype: str,
161
max_tokens: int,
162
num_logprobs: int,
138
- tensor_parallel_size: int,
139
- distributed_executor_backend: Optional[str] = None,
163
+ **kwargs,
164
):
165
with vllm_runner(model,
166
dtype=dtype,
143
- tensor_parallel_size=tensor_parallel_size,
144
- distributed_executor_backend=distributed_executor_backend,
167
enforce_eager=True,
168
limit_mm_per_prompt={
169
"audio":
170
max((len(audio) for _, audio in prompts_and_audios))
149
- }) as vllm_model:
171
+ },
172
+ **kwargs) as vllm_model:
173
vllm_outputs = vllm_model.generate_greedy_logprobs(
174
[prompt for prompt, _ in prompts_and_audios],
175
max_tokens,
@@ -162,8 +185,9 @@ def run_multi_audio_test(
185
@pytest.mark.parametrize("dtype", ["half"])
186
@pytest.mark.parametrize("max_tokens", [128])
187
@pytest.mark.parametrize("num_logprobs", [5])
188
+@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
189
def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
166
- num_logprobs: int) -> None:
190
+ num_logprobs: int, vllm_kwargs: dict) -> None:
191
192
vllm_prompt = _get_prompt(1, "Describe the audio above.", VLLM_PLACEHOLDER)
193
hf_prompt = _get_prompt(1, "Describe the audio above.", HF_PLACEHOLDER)
@@ -175,7 +199,7 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
199
dtype=dtype,
200
max_tokens=max_tokens,
201
num_logprobs=num_logprobs,
178
- tensor_parallel_size=1,
202
+ **vllm_kwargs,
203
)
204
205
@@ -183,9 +207,10 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
207
@pytest.mark.parametrize("dtype", ["half"])
208
@pytest.mark.parametrize("max_tokens", [128])
209
@pytest.mark.parametrize("num_logprobs", [5])
210
+@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
211
def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
187
- max_tokens: int,
188
- num_logprobs: int) -> None:
212
+ max_tokens: int, num_logprobs: int,
213
+ vllm_kwargs: dict) -> None:
214
215
vllm_prompt = _get_prompt(len(audio_assets),
216
"Describe each of the audios above.",
@@ -198,5 +223,37 @@ def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
223
dtype=dtype,
224
max_tokens=max_tokens,
225
num_logprobs=num_logprobs,
201
- tensor_parallel_size=1,
226
+ **vllm_kwargs,
227
)
228
+
229
+
230
+@pytest.mark.asyncio
231
+async def test_online_inference(client, audio_assets):
232
+ """Exercises online inference with/without chunked prefill enabled."""
233
+
234
+ messages = [{
235
+ "role":
236
+ "user",
237
+ "content": [
238
+ *[{
239
+ "type": "audio_url",
240
+ "audio_url": {
241
+ "url": audio.url
242
+ }
243
+ } for audio in audio_assets],
244
+ {
245
+ "type":
246
+ "text",
247
+ "text":
248
+ f"What's happening in these {len(audio_assets)} audio clips?"
249
+ },
250
+ ],
251
+ }]
252
+
253
+ chat_completion = await client.chat.completions.create(model=MODEL_NAME,
254
+ messages=messages,
255
+ max_tokens=10)
256
+
257
+ assert len(chat_completion.choices) == 1
258
+ choice = chat_completion.choices[0]
259
+ assert choice.finish_reason == "length"
tests/multimodal/test_processor_kwargs.py
+7
-7
@@ -5,8 +5,8 @@ from unittest.mock import patch
5
import pytest
6
import torch
7
8
-from vllm.inputs import DecoderOnlyInputs, InputContext, token_inputs
9
-from vllm.inputs.registry import InputRegistry
8
+from vllm.inputs import (DecoderOnlyInputs, DummyData, InputContext,
9
+ InputRegistry, token_inputs)
10
from vllm.multimodal import MultiModalRegistry
11
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12
@@ -56,7 +56,7 @@ def use_dummy_data_mock():
56
num_crops=DEFAULT_NUM_CROPS):
57
seq_data = SequenceData(
58
array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
59
- return seq_data, None
59
+ return DummyData(seq_data, None)
60
61
with patch(
62
"vllm.inputs.registry.InputRegistry._default_dummy_data_factory",
@@ -177,9 +177,9 @@ def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
177
# NOTE: seq_len is thrown away here since this will leverage the
178
# default dummy data factory that we have patched in, whose seq
179
# len is solely dependent on the value of the mm_processor_kwargs.
180
- seq_data, _ = dummy_registry.dummy_data_for_profiling(
180
+ dummy_data = dummy_registry.dummy_data_for_profiling(
181
ctx.model_config, seq_len=-1, mm_registry=mm_registry)
182
- assert len(seq_data.prompt_token_ids) == expected_seq_count
182
+ assert len(dummy_data.seq_data.prompt_token_ids) == expected_seq_count
183
184
185
@pytest.mark.parametrize(
@@ -206,9 +206,9 @@ def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
206
# NOTE: seq_len is thrown away here since this will leverage the
207
# default dummy data factory that we have patched in, whose seq
208
# len is solely dependent on the value of the mm_processor_kwargs.
209
- seq_data, _ = dummy_registry.dummy_data_for_profiling(
209
+ dummy_data = dummy_registry.dummy_data_for_profiling(
210
ctx.model_config, seq_len=-1, mm_registry=mm_registry)
211
- assert len(seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
211
+ assert len(dummy_data.seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
212
213
214
### Test overrides for the max token count per multimodal instance
tests/multimodal/test_utils.py
+45
-12
@@ -92,18 +92,50 @@ def test_repeat_and_pad_placeholder_tokens(model):
92
tokenizer = AutoTokenizer.from_pretrained(model)
93
94
test_cases = [
95
- ("<image>", 2, "<image><image>", [32000, 32000]),
96
- ("<image><image>", 2, "<image><image><image>", [32000, 32000, 32000]),
97
- ("<image><image>", [3, 2], "<image><image><image><image><image>",
98
- [32000, 32000, 32000, 32000, 32000]),
99
- ("Image:<image>Image:<image>!", [3, 2],
100
- "Image:<image><image><image>Image:<image><image>!",
101
- [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918]),
102
- ("<image>", [3, 2], "<image><image><image>", [32000, 32000, 32000]),
103
- ]
104
-
105
- for prompt, repeat_count, expected_prompt, expected_token_ids in test_cases:
106
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
95
+ (
96
+ "<image>",
97
+ 2,
98
+ "<image><image>",
99
+ [32000, 32000],
100
+ [{ "offset": 0, "length": 2 }],
101
+ ),
102
+ (
103
+ "<image><image>",
104
+ 2,
105
+ "<image><image><image>",
106
+ [32000, 32000, 32000],
107
+ [{ "offset": 0, "length": 2 }]),
108
+ (
109
+ "<image><image>",
110
+ [3, 2],
111
+ "<image><image><image><image><image>",
112
+ [32000, 32000, 32000, 32000, 32000],
113
+ [{ "offset": 0, "length": 3 }, { "offset": 3, "length": 2 }],
114
+ ),
115
+ (
116
+ "Image:<image>Image:<image>!",
117
+ [3, 2],
118
+ "Image:<image><image><image>Image:<image><image>!",
119
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
120
+ [{ "offset": 2, "length": 3 }, { "offset": 7, "length": 2 }],
121
+ ),
122
+ (
123
+ "<image>",
124
+ [3, 2],
125
+ "<image><image><image>",
126
+ [32000, 32000, 32000],
127
+ [{ "offset": 0, "length": 3 }],
128
+ ),
129
+ ] # yapf: disable
130
+
131
+ for (
132
+ prompt,
133
+ repeat_count,
134
+ expected_prompt,
135
+ expected_token_ids,
136
+ expected_ranges,
137
+ ) in test_cases:
138
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
139
tokenizer=tokenizer,
140
prompt=prompt,
141
prompt_token_ids=tokenizer.encode(prompt,
@@ -113,3 +145,4 @@ def test_repeat_and_pad_placeholder_tokens(model):
145
)
146
assert new_prompt == expected_prompt
147
assert new_token_ids == expected_token_ids
148
+ assert ranges == expected_ranges
tests/worker/test_model_input.py
+3
@@ -73,6 +73,7 @@ def test_model_runner_input():
73
num_prefill_tokens=2,
74
num_decode_tokens=3,
75
slot_mapping=torch.zeros(1),
76
+ multi_modal_placeholder_index_maps=None,
77
)
78
model_input = ModelInputForGPUWithSamplingMetadata(
79
input_tokens=torch.ones(10),
@@ -124,6 +125,7 @@ def test_embedding_model_runner_input():
125
num_prefill_tokens=2,
126
num_decode_tokens=3,
127
slot_mapping=torch.zeros(1),
128
+ multi_modal_placeholder_index_maps=None,
129
)
130
model_input = ModelInputForGPUWithPoolingMetadata(
131
input_tokens=torch.ones(10),
@@ -174,6 +176,7 @@ def test_multi_step_model_runner_input():
176
num_prefill_tokens=2,
177
num_decode_tokens=3,
178
slot_mapping=torch.zeros(1),
179
+ multi_modal_placeholder_index_maps=None,
180
)
181
frozen_model_input = ModelInputForGPUWithSamplingMetadata(
182
input_tokens=torch.ones(10),
vllm/attention/backends/abstract.py
+11
@@ -7,6 +7,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
7
8
import torch
9
10
+from vllm.multimodal import MultiModalPlaceholderMap
11
+
12
if TYPE_CHECKING:
13
from vllm.worker.model_runner_base import (ModelRunnerBase,
14
ModelRunnerInputBase,
@@ -108,6 +110,15 @@ class AttentionMetadata:
110
# in block 0, and 1st slot in block 1, respectively.
111
slot_mapping: torch.Tensor
112
113
+ # The index maps that relate multi-modal embeddings to the corresponding
114
+ # placeholders.
115
+ #
116
+ # N.B. These aren't really related to attention and don't belong on this
117
+ # type -- this is just a temporary solution to make them available to
118
+ # `model_executable`.
119
+ multi_modal_placeholder_index_maps: Optional[Dict[
120
+ str, MultiModalPlaceholderMap.IndexMap]]
121
+
122
@property
123
@abstractmethod
124
def prefill_metadata(self) -> Optional["AttentionMetadata"]:
vllm/attention/backends/blocksparse_attn.py
+3
@@ -215,6 +215,8 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
215
num_prefill_tokens=self.num_prefill_tokens,
216
num_decode_tokens=0,
217
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
218
+ multi_modal_placeholder_index_maps=self.
219
+ multi_modal_placeholder_index_maps,
220
seq_lens=self.seq_lens[:self.num_prefills],
221
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
222
max_query_len=self.max_query_len,
@@ -243,6 +245,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
245
num_prefill_tokens=0,
246
num_decode_tokens=self.num_decode_tokens,
247
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
248
+ multi_modal_placeholder_index_maps=None,
249
seq_lens=None,
250
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
251
max_query_len=None,
vllm/attention/backends/flash_attn.py
+20
@@ -1,4 +1,5 @@
1
"""Attention layer with FlashAttention."""
2
+from collections import defaultdict
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
5
@@ -14,6 +15,7 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
15
compute_slot_mapping_start_idx,
16
is_block_tables_empty)
17
from vllm.forward_context import get_forward_context
18
+from vllm.multimodal import MultiModalPlaceholderMap
19
from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
20
make_tensor_with_pad)
21
@@ -169,6 +171,8 @@ class FlashAttentionMetadata(AttentionMetadata):
171
num_prefill_tokens=self.num_prefill_tokens,
172
num_decode_tokens=0,
173
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
174
+ multi_modal_placeholder_index_maps=self.
175
+ multi_modal_placeholder_index_maps,
176
seq_lens=self.seq_lens[:self.num_prefills],
177
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
178
max_query_len=self.max_query_len,
@@ -198,6 +202,7 @@ class FlashAttentionMetadata(AttentionMetadata):
202
num_prefill_tokens=0,
203
num_decode_tokens=self.num_decode_tokens,
204
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
205
+ multi_modal_placeholder_index_maps=None,
206
seq_lens=None,
207
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
208
max_decode_query_len=self.max_decode_query_len,
@@ -297,6 +302,9 @@ class FlashAttentionMetadataBuilder(
302
self.context_lens: List[int] = []
303
self.block_tables: List[List[int]] = []
304
self.curr_seq_lens: List[int] = []
305
+ self.multimodal_placeholder_maps: Dict[
306
+ str,
307
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
308
self.num_prefills = 0
309
self.num_prefill_tokens = 0
310
self.num_decode_tokens = 0
@@ -327,6 +335,12 @@ class FlashAttentionMetadataBuilder(
335
self.context_lens.append(context_len)
336
337
if is_prompt:
338
+ mm_maps = inter_data.multi_modal_placeholder_maps
339
+ if mm_maps:
340
+ for modality, placeholders in mm_maps.items():
341
+ self.multimodal_placeholder_maps[modality].extend(
342
+ placeholders)
343
+
344
self.num_prefills += 1
345
self.num_prefill_tokens += token_len
346
self.prefill_seq_lens.append(seq_len)
@@ -449,6 +463,11 @@ class FlashAttentionMetadataBuilder(
463
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
464
dtype=torch.int32,
465
device=device)
466
+ placeholder_index_maps = {
467
+ modality: placeholder_map.index_map()
468
+ for modality, placeholder_map in
469
+ self.multimodal_placeholder_maps.items()
470
+ }
471
torch.cumsum(seq_lens_tensor,
472
dim=0,
473
dtype=seq_start_loc.dtype,
@@ -464,6 +483,7 @@ class FlashAttentionMetadataBuilder(
483
num_prefill_tokens=self.num_prefill_tokens,
484
num_decode_tokens=num_decode_tokens,
485
seq_lens=seq_lens,
486
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
487
seq_lens_tensor=seq_lens_tensor,
488
max_query_len=max_query_len,
489
max_decode_query_len=max_decode_query_len,
vllm/attention/backends/flashinfer.py
+18
@@ -1,7 +1,10 @@
1
+from collections import defaultdict
2
from contextlib import contextmanager
3
from dataclasses import dataclass
4
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
5
6
+from vllm.multimodal import MultiModalPlaceholderMap
7
+
8
try:
9
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
10
from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
@@ -215,6 +218,7 @@ class FlashInferState(AttentionState):
218
attn_metadata = self.runner.attn_backend.make_metadata(
219
num_prefills=0,
220
slot_mapping=self._graph_slot_mapping[:batch_size],
221
+ multi_modal_placeholder_index_maps=None,
222
num_prefill_tokens=0,
223
num_decode_tokens=batch_size,
224
max_prefill_seq_len=0,
@@ -470,6 +474,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
474
self.context_lens: List[int] = []
475
self.block_tables: List[List[int]] = []
476
self.curr_seq_lens: List[int] = []
477
+ self.multimodal_placeholder_maps: Dict[
478
+ str,
479
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
480
self.num_prefills = 0
481
self.num_prefill_tokens = 0
482
self.num_decode_tokens = 0
@@ -519,6 +526,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
526
inter_data.curr_sliding_window_blocks):
527
self.context_lens.append(context_len)
528
if is_prompt:
529
+ mm_maps = inter_data.multi_modal_placeholder_maps
530
+ if mm_maps:
531
+ for modality, placeholders in mm_maps.items():
532
+ self.multimodal_placeholder_maps[modality].extend(
533
+ placeholders)
534
self.num_prefills += 1
535
self.num_prefill_tokens += token_len
536
self.prefill_seq_lens.append(seq_len)
@@ -651,6 +663,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
663
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
664
dtype=torch.int32,
665
device=device)
666
+ placeholder_index_maps = {
667
+ modality: placeholder_map.index_map()
668
+ for modality, placeholder_map in
669
+ self.multimodal_placeholder_maps.items()
670
+ }
671
torch.cumsum(seq_lens_tensor,
672
dim=0,
673
dtype=seq_start_loc.dtype,
@@ -694,6 +711,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
711
decode_query_len=decode_query_len,
712
num_prefills=self.num_prefills,
713
slot_mapping=slot_mapping_tensor,
714
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
715
num_prefill_tokens=self.num_prefill_tokens,
716
num_decode_tokens=num_decode_tokens,
717
max_prefill_seq_len=max_prefill_seq_len,
vllm/attention/backends/placeholder_attn.py
+21
-1
@@ -1,5 +1,6 @@
1
+from collections import defaultdict
2
from dataclasses import dataclass
2
-from typing import TYPE_CHECKING, List, Optional, Tuple, Type
3
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type
4
5
import torch
6
@@ -7,6 +8,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
AttentionMetadata,
9
AttentionMetadataBuilder)
10
from vllm.attention.backends.utils import CommonAttentionState
11
+from vllm.multimodal import MultiModalPlaceholderMap
12
13
if TYPE_CHECKING:
14
from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -135,6 +137,8 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
137
num_prefill_tokens=self.num_prefill_tokens,
138
num_decode_tokens=0,
139
slot_mapping=slot_mapping,
140
+ multi_modal_placeholder_index_maps=self.
141
+ multi_modal_placeholder_index_maps,
142
seq_lens=self.seq_lens[:self.num_prefills],
143
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
144
max_decode_query_len=0,
@@ -167,6 +171,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
171
num_prefill_tokens=0,
172
num_decode_tokens=self.num_decode_tokens,
173
slot_mapping=slot_mapping,
174
+ multi_modal_placeholder_index_maps=None,
175
seq_lens=None,
176
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
177
max_decode_query_len=self.max_decode_query_len,
@@ -189,6 +194,9 @@ class PlaceholderAttentionMetadataBuilder(
194
self.prefill_seq_lens: List[int] = []
195
self.context_lens: List[int] = []
196
self.curr_seq_lens: List[int] = []
197
+ self.multimodal_placeholder_maps: Dict[
198
+ str,
199
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
200
self.num_prefills = 0
201
self.num_prefill_tokens = 0
202
self.num_decode_tokens = 0
@@ -213,6 +221,12 @@ class PlaceholderAttentionMetadataBuilder(
221
self.context_lens.append(context_len)
222
223
if is_prompt:
224
+ mm_maps = inter_data.multi_modal_placeholder_maps
225
+ if mm_maps:
226
+ for modality, placeholders in mm_maps.items():
227
+ self.multimodal_placeholder_maps[modality].extend(
228
+ placeholders)
229
+
230
self.num_prefills += 1
231
self.num_prefill_tokens += token_len
232
self.prefill_seq_lens.append(seq_len)
@@ -280,6 +294,11 @@ class PlaceholderAttentionMetadataBuilder(
294
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
295
dtype=torch.int32,
296
device=device)
297
+ placeholder_index_maps = {
298
+ modality: placeholder_map.index_map()
299
+ for modality, placeholder_map in
300
+ self.multimodal_placeholder_maps.items()
301
+ }
302
torch.cumsum(seq_lens_tensor,
303
dim=0,
304
dtype=seq_start_loc.dtype,
@@ -296,6 +315,7 @@ class PlaceholderAttentionMetadataBuilder(
315
return PlaceholderAttentionMetadata(
316
num_prefills=self.num_prefills,
317
slot_mapping=slot_mapping,
318
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
319
num_prefill_tokens=self.num_prefill_tokens,
320
num_decode_tokens=num_decode_tokens,
321
seq_lens=seq_lens,
vllm/attention/backends/rocm_flash_attn.py
+3
@@ -150,6 +150,8 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
150
num_prefill_tokens=self.num_prefill_tokens,
151
num_decode_tokens=0,
152
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
153
+ multi_modal_placeholder_index_maps=self.
154
+ multi_modal_placeholder_index_maps,
155
seq_lens=self.seq_lens[:self.num_prefills],
156
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
157
max_query_len=self.max_query_len,
@@ -178,6 +180,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
180
num_prefill_tokens=0,
181
num_decode_tokens=self.num_decode_tokens,
182
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
183
+ multi_modal_placeholder_index_maps=None,
184
seq_lens=None,
185
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
186
max_query_len=None,
vllm/attention/backends/utils.py
+18
@@ -1,4 +1,5 @@
1
"""Attention backend utils"""
2
+from collections import defaultdict
3
from contextlib import contextmanager
4
from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
5
@@ -7,6 +8,7 @@ import torch
8
9
from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
10
AttentionState)
11
+from vllm.multimodal import MultiModalPlaceholderMap
12
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
13
14
if TYPE_CHECKING:
@@ -123,6 +125,9 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
125
self.context_lens: List[int] = []
126
self.block_tables: List[List[int]] = []
127
self.curr_seq_lens: List[int] = []
128
+ self.multimodal_placeholder_maps: Dict[
129
+ str,
130
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
131
self.num_prefills = 0
132
self.num_prefill_tokens = 0
133
self.num_decode_tokens = 0
@@ -147,6 +152,12 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
152
inter_data.curr_sliding_window_blocks):
153
self.context_lens.append(context_len)
154
if is_prompt:
155
+ mm_maps = inter_data.multi_modal_placeholder_maps
156
+ if mm_maps:
157
+ for modality, placeholders in mm_maps.items():
158
+ self.multimodal_placeholder_maps[modality].extend(
159
+ placeholders)
160
+
161
self.num_prefills += 1
162
self.num_prefill_tokens += token_len
163
self.prefill_seq_lens.append(seq_len)
@@ -242,6 +253,11 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
253
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
254
dtype=torch.int32,
255
device=device)
256
+ placeholder_index_maps = {
257
+ modality: placeholder_map.index_map()
258
+ for modality, placeholder_map in
259
+ self.multimodal_placeholder_maps.items()
260
+ }
261
torch.cumsum(seq_lens_tensor,
262
dim=0,
263
dtype=seq_start_loc.dtype,
@@ -254,6 +270,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
270
return self._metadata_cls( # type: ignore
271
num_prefills=self.num_prefills,
272
slot_mapping=slot_mapping_tensor,
273
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
274
num_prefill_tokens=self.num_prefill_tokens,
275
num_decode_tokens=num_decode_tokens,
276
seq_lens=seq_lens,
@@ -305,6 +322,7 @@ class CommonAttentionState(AttentionState):
322
num_prefill_tokens=0,
323
num_decode_tokens=batch_size,
324
slot_mapping=self._graph_slot_mapping[:batch_size],
325
+ multi_modal_placeholder_index_maps=None,
326
seq_lens=None,
327
seq_lens_tensor=self._graph_seq_lens[:batch_size],
328
max_query_len=1,
vllm/attention/backends/xformers.py
+3
@@ -212,6 +212,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
212
num_prefill_tokens=self.num_prefill_tokens,
213
num_decode_tokens=0,
214
slot_mapping=slot_mapping,
215
+ multi_modal_placeholder_index_maps=self.
216
+ multi_modal_placeholder_index_maps,
217
seq_lens=seq_lens,
218
seq_lens_tensor=seq_lens_tensor,
219
max_query_len=self.max_query_len,
@@ -255,6 +257,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
257
num_prefill_tokens=0,
258
num_decode_tokens=self.num_decode_tokens,
259
slot_mapping=slot_mapping,
260
+ multi_modal_placeholder_index_maps=None,
261
seq_lens_tensor=seq_lens_tensor,
262
max_prefill_seq_len=0,
263
max_decode_seq_len=self.max_decode_seq_len,
vllm/core/scheduler.py
+2
@@ -1308,6 +1308,8 @@ class Scheduler:
1308
# `multi_modal_data` will be None.
1309
multi_modal_data=seq_group.multi_modal_data
1310
if scheduler_outputs.num_prefill_groups > 0 else None,
1311
+ multi_modal_placeholders=seq_group.multi_modal_placeholders
1312
+ if scheduler_outputs.num_prefill_groups > 0 else None,
1313
mm_processor_kwargs=seq_group.mm_processor_kwargs,
1314
prompt_adapter_request=seq_group.prompt_adapter_request,
1315
)
vllm/inputs/__init__.py
+2
-1
@@ -3,7 +3,7 @@ from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
3
SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4
build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5
token_inputs, zip_enc_dec_prompts)
6
-from .registry import InputContext, InputRegistry
6
+from .registry import DummyData, InputContext, InputRegistry
7
8
INPUT_REGISTRY = InputRegistry()
9
"""
@@ -29,6 +29,7 @@ __all__ = [
29
"to_enc_dec_tuple_list",
30
"zip_enc_dec_prompts",
31
"INPUT_REGISTRY",
32
+ "DummyData",
33
"InputContext",
34
"InputRegistry",
35
]
vllm/inputs/data.py
+10
-1
@@ -4,7 +4,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
4
from typing_extensions import NotRequired, TypedDict, TypeVar
5
6
if TYPE_CHECKING:
7
- from vllm.multimodal import MultiModalDataDict
7
+ from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
8
9
10
class TextPrompt(TypedDict):
@@ -136,6 +136,12 @@ class TokenInputs(TypedDict):
136
if the model supports it.
137
"""
138
139
+ multi_modal_placeholders: NotRequired[
140
+ Optional["MultiModalPlaceholderDict"]]
141
+ """
142
+ Placeholder ranges for the multi-modal data.
143
+ """
144
+
145
mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
146
"""
147
Optional multi-modal processor kwargs to be forwarded to the
@@ -149,6 +155,7 @@ def token_inputs(
155
prompt_token_ids: List[int],
156
prompt: Optional[str] = None,
157
multi_modal_data: Optional["MultiModalDataDict"] = None,
158
+ multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
159
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160
) -> TokenInputs:
161
"""Construct :class:`TokenInputs` from optional values."""
@@ -158,6 +165,8 @@ def token_inputs(
165
inputs["prompt"] = prompt
166
if multi_modal_data is not None:
167
inputs["multi_modal_data"] = multi_modal_data
168
+ if multi_modal_placeholders is not None:
169
+ inputs["multi_modal_placeholders"] = multi_modal_placeholders
170
if mm_processor_kwargs is not None:
171
inputs["mm_processor_kwargs"] = mm_processor_kwargs
172
vllm/inputs/registry.py
+23
-17
@@ -1,8 +1,8 @@
1
import functools
2
from collections import UserDict
3
from dataclasses import dataclass
4
-from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
5
- Protocol, Tuple, Type)
4
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5
+ Optional, Protocol, Type)
6
7
from torch import nn
8
from transformers import PretrainedConfig
@@ -16,7 +16,8 @@ from .data import DecoderOnlyInputs
16
17
if TYPE_CHECKING:
18
from vllm.config import ModelConfig
19
- from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
19
+ from vllm.multimodal import (MultiModalDataDict, MultiModalPlaceholderDict,
20
+ MultiModalRegistry)
21
from vllm.sequence import SequenceData
22
23
logger = init_logger(__name__)
@@ -63,6 +64,14 @@ class InputContext:
64
N = TypeVar("N", bound=Type[nn.Module])
65
66
67
+class DummyData(NamedTuple):
68
+ """Dummy data used for profiling."""
69
+
70
+ seq_data: "SequenceData"
71
+ multi_modal_data: Optional["MultiModalDataDict"] = None
72
+ multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None
73
+
74
+
75
class DummyDataFactory(Protocol):
76
77
def __call__(
@@ -71,7 +80,7 @@ class DummyDataFactory(Protocol):
80
seq_len: int,
81
mm_counts: Mapping[str, int],
82
**mm_processor_kwargs: Any,
74
- ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
83
+ ) -> DummyData:
84
"""
85
Create dummy data to be inputted into the model.
86
@@ -123,7 +132,7 @@ class InputRegistry:
132
ctx: InputContext,
133
seq_len: int,
134
mm_counts: Mapping[str, int],
126
- ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
135
+ ) -> DummyData:
136
"""
137
The default dummy data factory represents the longest possible text
138
that can be inputted to the model.
@@ -134,10 +143,7 @@ class InputRegistry:
143
# Avoid circular import
144
from vllm.sequence import SequenceData
145
137
- dummy_seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
138
- dummy_multi_modal_data = None
139
-
140
- return dummy_seq_data, dummy_multi_modal_data
146
+ return DummyData(SequenceData.from_prompt_token_counts((0, seq_len)))
147
148
def register_dummy_data(self, factory: DummyDataFactory):
149
"""
@@ -195,7 +201,7 @@ class InputRegistry:
201
seq_len: int,
202
mm_registry: "MultiModalRegistry",
203
is_encoder_data: bool = False,
198
- ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
204
+ ) -> DummyData:
205
"""
206
Create dummy data for profiling the memory usage of a model.
207
@@ -220,12 +226,12 @@ class InputRegistry:
226
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
227
dummy_factory, overrides=model_config.mm_processor_kwargs)
228
223
- seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len,
224
- _MultiModalCounts(mm_counts),
225
- **mm_processor_kwargs)
229
+ dummy_data = dummy_factory(InputContext(model_config), seq_len,
230
+ _MultiModalCounts(mm_counts),
231
+ **mm_processor_kwargs)
232
233
# Having more tokens is over-conservative but otherwise fine
228
- num_tokens = seq_data.prompt_token_ids
234
+ num_tokens = dummy_data.seq_data.prompt_token_ids
235
if len(num_tokens) < seq_len:
236
if is_encoder_data:
237
print_warning_once(
@@ -235,15 +241,15 @@ class InputRegistry:
241
raise AssertionError(
242
f"Expected at least {seq_len} dummy tokens for profiling, "
243
f"but found {len(num_tokens)} tokens instead.")
238
- if mm_data is not None:
239
- for k, v in mm_data.items():
244
+ if dummy_data.multi_modal_data is not None:
245
+ for k, v in dummy_data.multi_modal_data.items():
246
num_items = len(v) if isinstance(v, list) else 1
247
num_expected = mm_counts[k]
248
assert num_items >= num_expected, (
249
f"Expected at least {num_expected} dummy '{k}' instances "
250
f"for profiling, but found {num_items} instances instead.")
251
246
- return seq_data, mm_data
252
+ return dummy_data
253
254
def _default_input_processor(
255
self,
vllm/model_executor/models/blip.py
+8
-2
@@ -98,6 +98,11 @@ def input_processor_for_blip(
98
if multi_modal_data is None or "image" not in multi_modal_data:
99
return inputs
100
101
+ if "multi_modal_placeholders" in inputs and "image" in inputs[
102
+ "multi_modal_placeholders"]:
103
+ # The inputs already have placeholders.
104
+ return inputs
105
+
106
tokenizer = cached_get_tokenizer(model_config.tokenizer)
107
108
if image_feature_size_override is None:
@@ -105,7 +110,7 @@ def input_processor_for_blip(
110
else:
111
image_feature_size = image_feature_size_override
112
108
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
113
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
114
tokenizer,
115
inputs.get("prompt"),
116
inputs["prompt_token_ids"],
@@ -116,7 +121,8 @@ def input_processor_for_blip(
121
# NOTE: Create a defensive copy of the original inputs
122
return token_inputs(prompt_token_ids=new_token_ids,
123
prompt=new_prompt,
119
- multi_modal_data=multi_modal_data)
124
+ multi_modal_data=multi_modal_data,
125
+ multi_modal_placeholders={"image": ranges})
126
127
128
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
vllm/model_executor/models/blip2.py
+10
-5
@@ -9,13 +9,14 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, MultiModalConfig
12
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
13
- token_inputs)
12
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13
+ InputContext, token_inputs)
14
from vllm.model_executor.layers.activation import get_act_fn
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
+from vllm.multimodal.utils import consecutive_placeholder_ranges
20
from vllm.sequence import IntermediateTensors, SequenceData
21
22
from .blip import (BlipVisionModel, dummy_image_for_blip,
@@ -425,7 +426,11 @@ def dummy_seq_data_for_blip2(
426
return SequenceData.from_prompt_token_counts(
427
(image_token_id, image_feature_size * num_images),
428
(0, seq_len - image_feature_size * num_images),
428
- )
429
+ ), {
430
+ "image":
431
+ consecutive_placeholder_ranges(num_items=num_images,
432
+ item_size=image_feature_size)
433
+ }
434
435
436
def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
@@ -434,7 +439,7 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
439
vision_config = hf_config.vision_config
440
num_images = mm_counts["image"]
441
437
- seq_data = dummy_seq_data_for_blip2(
442
+ seq_data, ranges = dummy_seq_data_for_blip2(
443
hf_config,
444
seq_len,
445
num_images,
@@ -444,7 +449,7 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
449
if isinstance(vision_config, Blip2VisionConfig):
450
mm_data = dummy_image_for_blip(vision_config, num_images)
451
447
- return seq_data, mm_data
452
+ return DummyData(seq_data, mm_data, ranges)
453
454
msg = f"Unsupported vision config: {type(vision_config)}"
455
raise NotImplementedError(msg)
vllm/model_executor/models/chameleon.py
+16
-6
@@ -11,8 +11,8 @@ from transformers import ChameleonConfig, ChameleonVQVAEConfig
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
13
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15
- token_inputs)
14
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
+ InputContext, token_inputs)
16
from vllm.model_executor.layers.activation import SiluAndMul
17
from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -30,6 +30,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
30
from vllm.model_executor.utils import set_weight_attrs
31
from vllm.multimodal import MULTIMODAL_REGISTRY
32
from vllm.multimodal.utils import (cached_get_tokenizer,
33
+ consecutive_placeholder_ranges,
34
repeat_and_pad_placeholder_tokens)
35
from vllm.sequence import IntermediateTensors, SequenceData
36
from vllm.utils import print_warning_once
@@ -73,7 +74,11 @@ def dummy_seq_data_for_chameleon(
74
return SequenceData.from_prompt_token_counts(
75
(image_token_id, image_feature_size * num_images),
76
(0, seq_len - image_feature_size * num_images),
76
- )
77
+ ), {
78
+ "image":
79
+ consecutive_placeholder_ranges(num_items=num_images,
80
+ item_size=image_feature_size)
81
+ }
82
83
84
def dummy_image_for_chameleon(
@@ -97,14 +102,14 @@ def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
102
mm_counts: Mapping[str, int]):
103
num_images = mm_counts["image"]
104
100
- seq_data = dummy_seq_data_for_chameleon(
105
+ seq_data, ranges = dummy_seq_data_for_chameleon(
106
seq_len,
107
num_images,
108
image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
109
)
110
111
mm_data = dummy_image_for_chameleon(num_images)
107
- return seq_data, mm_data
112
+ return DummyData(seq_data, mm_data, ranges)
113
114
115
def input_processor_for_chameleon(ctx: InputContext,
@@ -120,9 +125,14 @@ def input_processor_for_chameleon(ctx: InputContext,
125
if multi_modal_data is None or "image" not in multi_modal_data:
126
return inputs
127
128
+ if "multi_modal_placeholders" in inputs and "image" in inputs[
129
+ "multi_modal_placeholders"]:
130
+ # The inputs already have placeholders.
131
+ return inputs
132
+
133
model_config = ctx.model_config
134
tokenizer = cached_get_tokenizer(model_config.tokenizer)
125
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
135
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
136
tokenizer,
137
inputs.get("prompt"),
138
inputs["prompt_token_ids"],
vllm/model_executor/models/clip.py
+21
-11
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
from vllm.multimodal.utils import (cached_get_tokenizer,
22
+ consecutive_placeholder_ranges,
23
repeat_and_pad_placeholder_tokens)
24
from vllm.sequence import SequenceData
25
@@ -49,14 +50,13 @@ def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
50
return get_clip_image_feature_size(hf_config)
51
52
52
-def dummy_seq_data_for_clip(
53
- hf_config: CLIPVisionConfig,
54
- seq_len: int,
55
- num_images: int,
56
- *,
57
- image_token_id: int,
58
- image_feature_size_override: Optional[int] = None,
59
-):
53
+def dummy_seq_data_for_clip(hf_config: CLIPVisionConfig,
54
+ seq_len: int,
55
+ num_images: int,
56
+ *,
57
+ image_token_id: int,
58
+ image_feature_size_override: Optional[int] = None,
59
+ mm_key: str = "image"):
60
if image_feature_size_override is None:
61
image_feature_size = get_clip_image_feature_size(hf_config)
62
else:
@@ -65,7 +65,11 @@ def dummy_seq_data_for_clip(
65
return SequenceData.from_prompt_token_counts(
66
(image_token_id, image_feature_size * num_images),
67
(0, seq_len - image_feature_size * num_images),
68
- )
68
+ ), {
69
+ mm_key:
70
+ consecutive_placeholder_ranges(num_items=num_images,
71
+ item_size=image_feature_size)
72
+ }
73
74
75
def dummy_image_for_clip(
@@ -117,6 +121,11 @@ def input_processor_for_clip(
121
if multi_modal_data is None or "image" not in multi_modal_data:
122
return inputs
123
124
+ if "multi_modal_placeholders" in inputs and "image" in inputs[
125
+ "multi_modal_placeholders"]:
126
+ # The inputs already have placeholders.
127
+ return inputs
128
+
129
tokenizer = cached_get_tokenizer(model_config.tokenizer)
130
131
if image_feature_size_override is None:
@@ -130,7 +139,7 @@ def input_processor_for_clip(
139
else:
140
image_feature_size = image_feature_size_override
141
133
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
142
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
143
tokenizer,
144
inputs.get("prompt"),
145
inputs["prompt_token_ids"],
@@ -141,7 +150,8 @@ def input_processor_for_clip(
150
# NOTE: Create a defensive copy of the original inputs
151
return token_inputs(prompt_token_ids=new_token_ids,
152
prompt=new_prompt,
144
- multi_modal_data=multi_modal_data)
153
+ multi_modal_data=multi_modal_data,
154
+ multi_modal_placeholders={"image": ranges})
155
156
157
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/fuyu.py
+20
-11
@@ -27,8 +27,8 @@ from transformers import FuyuConfig, FuyuImageProcessor
27
28
from vllm.attention import AttentionMetadata
29
from vllm.config import CacheConfig, MultiModalConfig
30
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31
- token_inputs)
30
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31
+ InputContext, token_inputs)
32
from vllm.model_executor.layers.linear import ColumnParallelLinear
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -37,9 +37,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
38
from vllm.multimodal.base import MultiModalInputs
39
from vllm.multimodal.image import cached_get_image_processor
40
-from vllm.multimodal.utils import cached_get_tokenizer
40
+from vllm.multimodal.utils import (cached_get_tokenizer,
41
+ consecutive_placeholder_ranges)
42
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43
SequenceData)
44
+from vllm.utils import is_list_of
45
46
from .interfaces import SupportsMultiModal, SupportsPP
47
from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
@@ -103,7 +105,11 @@ def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
105
token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106
token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107
[0]) * (seq_len - image_feature_size * num_images)
106
- return SequenceData(token_ids)
108
+ return SequenceData(token_ids), {
109
+ "image":
110
+ consecutive_placeholder_ranges(num_items=num_images,
111
+ item_size=image_feature_size)
112
+ }
113
114
115
def dummy_image_for_fuyu(
@@ -119,15 +125,15 @@ def dummy_image_for_fuyu(
125
def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
126
mm_counts: Mapping[str, int]):
127
num_images = mm_counts["image"]
122
- seq_data = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
128
+ seq_data, ranges = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
129
mm_data = dummy_image_for_fuyu(num_images,
130
image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
131
image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
126
- return seq_data, mm_data
132
+ return DummyData(seq_data, mm_data, ranges)
133
134
135
def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
130
- data: Image.Image):
136
+ data: List[Image.Image]):
137
image_encoding = image_processor.preprocess(data, return_tensors="pt")
138
batch_images = torch.stack([img[0] for img in image_encoding["images"]
139
]).unsqueeze(1)
@@ -158,8 +164,10 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
164
model_config = ctx.model_config
165
image_data = multi_modal_data["image"]
166
new_multi_modal_data = {}
167
+ image_list = image_data if isinstance(image_data, list) else [image_data]
168
+
169
# process image data
162
- if isinstance(image_data, Image.Image):
170
+ if is_list_of(image_list, Image.Image):
171
# Fuyu's image_processor can also finish token padding
172
image_processor: FuyuImageProcessor = cached_get_image_processor(
173
model_config.model)
@@ -171,7 +179,7 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
179
])
180
new_multi_modal_data["image"] = image_patches
181
174
- elif isinstance(image_data, torch.Tensor):
182
+ elif is_list_of(image_list, torch.Tensor):
183
raise NotImplementedError("Embeddings input is not supported yet")
184
else:
185
raise TypeError(f"Invalid image type: {type(image_data)}")
@@ -198,12 +206,13 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
206
207
def input_mapper_for_fuyu(ctx: InputContext, data: object):
208
model_config = ctx.model_config
201
- if isinstance(data, Image.Image):
209
+ data_list = data if isinstance(data, list) else [data]
210
+ if is_list_of(data_list, Image.Image):
211
# Fuyu's image_processor can also finish token padding
212
image_processor: FuyuImageProcessor = cached_get_image_processor(
213
model_config.model)
214
206
- model_image_input = _fuyu_image_preprocess(image_processor, data)
215
+ model_image_input = _fuyu_image_preprocess(image_processor, data_list)
216
data = torch.stack([
217
image_patch[0]
218
for image_patch in model_image_input["image_patches"]
vllm/model_executor/models/internvl.py
+4
-4
@@ -17,8 +17,8 @@ from transformers import PretrainedConfig
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, MultiModalConfig
20
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21
- token_inputs)
20
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
+ InputContext, token_inputs)
22
from vllm.model_executor.layers.quantization import (AWQConfig,
23
QuantizationConfig)
24
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -379,7 +379,7 @@ class InternVLInputPipeline:
379
model_config.tokenizer,
380
trust_remote_code=model_config.trust_remote_code)
381
382
- seq_data = dummy_seq_data_for_clip(
382
+ seq_data, ranges = dummy_seq_data_for_clip(
383
hf_config.vision_config,
384
seq_len,
385
num_images,
@@ -398,7 +398,7 @@ class InternVLInputPipeline:
398
image_height_override=max_image_height,
399
)
400
401
- return seq_data, mm_data
401
+ return DummyData(seq_data, mm_data, ranges)
402
403
404
input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
vllm/model_executor/models/llava.py
+8
-7
@@ -10,7 +10,8 @@ from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
10
11
from vllm.attention import AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
13
-from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
13
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14
+ InputContext)
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.quantization import QuantizationConfig
17
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -111,7 +112,7 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
112
image_feature_size = get_max_llava_image_tokens(ctx)
113
114
if isinstance(vision_config, CLIPVisionConfig):
114
- seq_data = dummy_seq_data_for_clip(
115
+ seq_data, ranges = dummy_seq_data_for_clip(
116
vision_config,
117
seq_len,
118
num_images,
@@ -120,9 +121,9 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
121
)
122
123
mm_data = dummy_image_for_clip(vision_config, num_images)
123
- return seq_data, mm_data
124
+ return DummyData(seq_data, mm_data, ranges)
125
elif isinstance(vision_config, SiglipVisionConfig):
125
- seq_data = dummy_seq_data_for_siglip(
126
+ seq_data, ranges = dummy_seq_data_for_siglip(
127
vision_config,
128
seq_len,
129
num_images,
@@ -131,9 +132,9 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
132
)
133
134
mm_data = dummy_image_for_siglip(vision_config, num_images)
134
- return seq_data, mm_data
135
+ return DummyData(seq_data, mm_data, ranges)
136
elif isinstance(vision_config, PixtralVisionConfig):
136
- seq_data = dummy_seq_data_for_pixtral_hf(
137
+ seq_data, ranges = dummy_seq_data_for_pixtral_hf(
138
vision_config,
139
seq_len,
140
num_images,
@@ -142,7 +143,7 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
143
)
144
145
mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
145
- return seq_data, mm_data
146
+ return DummyData(seq_data, mm_data, ranges)
147
148
msg = f"Unsupported vision config: {type(vision_config)}"
149
raise NotImplementedError(msg)
vllm/model_executor/models/llava_next.py
+6
-5
@@ -12,7 +12,8 @@ from typing_extensions import NotRequired
12
13
from vllm.attention import AttentionMetadata
14
from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
15
-from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
15
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16
+ InputContext)
17
from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
from vllm.model_executor.layers.quantization import QuantizationConfig
19
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -180,7 +181,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
181
max_feat_height, max_feat_width = pinpoint
182
183
if isinstance(vision_config, CLIPVisionConfig):
183
- seq_data = dummy_seq_data_for_clip(
184
+ seq_data, ranges = dummy_seq_data_for_clip(
185
vision_config,
186
seq_len,
187
num_images,
@@ -195,9 +196,9 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
196
image_height_override=max_feat_height,
197
)
198
198
- return seq_data, mm_data
199
+ return DummyData(seq_data, mm_data, ranges)
200
elif isinstance(vision_config, SiglipVisionConfig):
200
- seq_data = dummy_seq_data_for_siglip(
201
+ seq_data, ranges = dummy_seq_data_for_siglip(
202
vision_config,
203
seq_len,
204
num_images,
@@ -212,7 +213,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
213
image_height_override=max_feat_height,
214
)
215
215
- return seq_data, mm_data
216
+ return DummyData(seq_data, mm_data, ranges)
217
218
msg = f"Unsupported vision config: {type(vision_config)}"
219
raise NotImplementedError(msg)
vllm/model_executor/models/llava_next_video.py
+17
-8
@@ -11,8 +11,8 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11
12
from vllm.attention import AttentionMetadata
13
from vllm.config import CacheConfig, MultiModalConfig
14
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15
- token_inputs)
14
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
+ InputContext, token_inputs)
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -108,33 +108,35 @@ def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
108
video_feature_size = frames_per_video * tokens_per_frame
109
110
if isinstance(vision_config, CLIPVisionConfig):
111
- seq_data = dummy_seq_data_for_clip(
111
+ seq_data, ranges = dummy_seq_data_for_clip(
112
vision_config,
113
seq_len,
114
num_videos,
115
image_token_id=hf_config.video_token_index,
116
image_feature_size_override=video_feature_size,
117
+ mm_key="video",
118
)
119
120
pil_frame = dummy_image_for_clip(vision_config, num_images=1)
121
np_frame = np.array(pil_frame["image"])
122
mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
123
mm_data = {"video": mm_data_per_video}
123
- return seq_data, mm_data
124
+ return DummyData(seq_data, mm_data, ranges)
125
elif isinstance(vision_config, SiglipVisionConfig):
125
- seq_data = dummy_seq_data_for_siglip(
126
+ seq_data, ranges = dummy_seq_data_for_siglip(
127
vision_config,
128
seq_len,
129
num_videos,
130
image_token_id=hf_config.video_token_index,
131
image_feature_size_override=video_feature_size,
132
+ mm_key="video",
133
)
134
135
pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
136
np_frame = np.array(pil_frame["image"])
137
mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
138
mm_data = {"video": mm_data_per_video}
137
- return seq_data, mm_data
139
+ return DummyData(seq_data, mm_data, ranges)
140
141
msg = f"Unsupported vision config: {type(vision_config)}"
142
raise NotImplementedError(msg)
@@ -145,6 +147,12 @@ def input_processor_for_llava_next_video(ctx: InputContext,
147
multi_modal_data = inputs.get("multi_modal_data")
148
if multi_modal_data is None or "video" not in multi_modal_data:
149
return inputs
150
+
151
+ if "multi_modal_placeholders" in inputs and "video" in inputs[
152
+ "multi_modal_placeholders"]:
153
+ # The inputs already have placeholders.
154
+ return inputs
155
+
156
video_data = multi_modal_data["video"]
157
158
model_config = ctx.model_config
@@ -160,7 +168,7 @@ def input_processor_for_llava_next_video(ctx: InputContext,
168
169
tokenizer = cached_get_tokenizer(model_config.tokenizer)
170
163
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
171
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
172
tokenizer,
173
inputs.get("prompt"),
174
inputs["prompt_token_ids"],
@@ -170,7 +178,8 @@ def input_processor_for_llava_next_video(ctx: InputContext,
178
179
return token_inputs(prompt_token_ids=new_token_ids,
180
prompt=new_prompt,
173
- multi_modal_data=multi_modal_data)
181
+ multi_modal_data=multi_modal_data,
182
+ multi_modal_placeholders={"video": ranges})
183
184
elif is_list_of(video_data, np.ndarray):
185
raise NotImplementedError(
vllm/model_executor/models/llava_onevision.py
+11
-10
@@ -15,8 +15,8 @@ from typing_extensions import NotRequired
15
16
from vllm.attention import AttentionMetadata
17
from vllm.config import CacheConfig, MultiModalConfig
18
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
19
- token_inputs)
18
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19
+ InputContext, token_inputs)
20
from vllm.model_executor.layers.activation import get_act_fn
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -218,31 +218,31 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
218
video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
219
220
if isinstance(vision_config, CLIPVisionConfig):
221
- seq_data = dummy_seq_data_for_clip(
221
+ seq_data, ranges = dummy_seq_data_for_clip(
222
vision_config,
223
seq_len,
224
num_videos,
225
image_token_id=hf_config.video_token_index,
226
image_feature_size_override=video_feature_size,
227
- )
227
+ mm_key="video")
228
229
mm_data = dummy_video_for_clip(vision_config,
230
num_frames=num_frames,
231
num_videos=num_videos)
232
- return seq_data, mm_data
232
+ return DummyData(seq_data, mm_data, ranges)
233
elif isinstance(vision_config, SiglipVisionConfig):
234
- seq_data = dummy_seq_data_for_siglip(
234
+ seq_data, ranges = dummy_seq_data_for_siglip(
235
vision_config,
236
seq_len,
237
num_videos,
238
image_token_id=hf_config.video_token_index,
239
image_feature_size_override=video_feature_size,
240
- )
240
+ mm_key="video")
241
242
mm_data = dummy_video_for_siglip(vision_config,
243
num_frames=num_frames,
244
num_videos=num_videos)
245
- return seq_data, mm_data
245
+ return DummyData(seq_data, mm_data, ranges)
246
247
msg = f"Unsupported vision config: {type(vision_config)}"
248
raise NotImplementedError(msg)
@@ -320,7 +320,7 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
320
video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
321
tokenizer = cached_get_tokenizer(model_config.tokenizer)
322
323
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
323
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
324
tokenizer,
325
inputs.get("prompt"),
326
inputs["prompt_token_ids"],
@@ -330,7 +330,8 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
330
331
return token_inputs(prompt_token_ids=new_token_ids,
332
prompt=new_prompt,
333
- multi_modal_data=multi_modal_data)
333
+ multi_modal_data=multi_modal_data,
334
+ multi_modal_placeholders={"video": ranges})
335
336
elif is_list_of(video_data, np.ndarray):
337
video_feature_size = []
vllm/model_executor/models/minicpmv.py
+3
-3
@@ -36,8 +36,8 @@ from typing_extensions import NotRequired
36
37
from vllm.attention import AttentionMetadata
38
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
40
- token_inputs)
39
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40
+ InputContext, token_inputs)
41
from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization import QuantizationConfig
43
from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
@@ -277,7 +277,7 @@ def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
277
seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
278
mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
279
280
- return seq_data, mm_data
280
+ return DummyData(seq_data, mm_data)
281
282
283
def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
vllm/model_executor/models/mllama.py
+4
-3
@@ -36,7 +36,7 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36
from vllm.attention.ops.paged_attn import PagedAttention
37
from vllm.config import CacheConfig, MultiModalConfig
38
from vllm.distributed import get_tensor_model_parallel_world_size
39
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
39
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40
EncoderDecoderInputs, InputContext)
41
from vllm.logger import init_logger
42
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -176,13 +176,14 @@ def dummy_image(num_images: int, ):
176
def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
177
mm_counts: Mapping[str, int]):
178
num_images = mm_counts["image"]
179
- return dummy_decoder_seq_data(seq_len, num_images), None
179
+ return DummyData(dummy_decoder_seq_data(seq_len, num_images))
180
181
182
def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
183
mm_counts: Mapping[str, int]):
184
num_images = mm_counts["image"]
185
- return dummy_encoder_seq_data(ctx, num_images), dummy_image(num_images)
185
+ return DummyData(dummy_encoder_seq_data(ctx, num_images),
186
+ dummy_image(num_images))
187
188
189
def _prepare_aspect_ratio_attention_mask(
vllm/model_executor/models/paligemma.py
+4
-4
@@ -7,8 +7,8 @@ from transformers import PaliGemmaConfig
7
8
from vllm.attention import AttentionMetadata
9
from vllm.config import CacheConfig, MultiModalConfig
10
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
11
- token_inputs)
10
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
11
+ InputContext, token_inputs)
12
from vllm.logger import init_logger
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -58,7 +58,7 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
58
vision_config = hf_config.vision_config
59
num_images = mm_counts["image"]
60
61
- seq_data = dummy_seq_data_for_siglip(
61
+ seq_data, ranges = dummy_seq_data_for_siglip(
62
vision_config,
63
seq_len,
64
num_images,
@@ -66,7 +66,7 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
66
)
67
68
mm_data = dummy_image_for_siglip(vision_config, num_images)
69
- return seq_data, mm_data
69
+ return DummyData(seq_data, mm_data, ranges)
70
71
72
def input_processor_for_paligemma(ctx: InputContext,
vllm/model_executor/models/phi3v.py
+4
-4
@@ -28,8 +28,8 @@ from transformers import CLIPVisionConfig, PretrainedConfig
28
from vllm.attention import AttentionMetadata
29
from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
30
PoolerConfig)
31
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
32
- token_inputs)
31
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
32
+ InputContext, token_inputs)
33
from vllm.logger import init_logger
34
from vllm.model_executor.layers.pooler import Pooler, PoolingType
35
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -380,7 +380,7 @@ def dummy_data_for_phi3v(ctx: InputContext,
380
381
image_feature_size = get_max_phi3v_image_tokens(ctx, num_crops=num_crops)
382
383
- seq_data = dummy_seq_data_for_clip(
383
+ seq_data, ranges = dummy_seq_data_for_clip(
384
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
385
seq_len,
386
num_images,
@@ -394,7 +394,7 @@ def dummy_data_for_phi3v(ctx: InputContext,
394
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
395
)
396
397
- return seq_data, mm_data
397
+ return DummyData(seq_data, mm_data, ranges)
398
399
400
@lru_cache
vllm/model_executor/models/pixtral.py
+22
-12
@@ -17,8 +17,8 @@ from transformers.models.pixtral.modeling_pixtral import (
17
18
from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
20
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21
- token_inputs)
20
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
+ InputContext, token_inputs)
22
from vllm.model_executor.layers.activation import get_act_fn
23
from vllm.model_executor.layers.layernorm import RMSNorm
24
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -28,7 +28,8 @@ from vllm.model_executor.models.utils import merge_multimodal_embeddings
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.base import MultiModalInputs
31
-from vllm.multimodal.utils import cached_get_tokenizer
31
+from vllm.multimodal.utils import (cached_get_tokenizer,
32
+ consecutive_placeholder_ranges)
33
from vllm.sequence import IntermediateTensors, SequenceData
34
from vllm.transformers_utils.processor import cached_get_processor
35
from vllm.utils import is_list_of
@@ -81,7 +82,12 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
82
)
83
84
mm_data = {"image": num_images * [image]}
84
- return seq_data, mm_data
85
+ mm_placeholders = {
86
+ "image":
87
+ consecutive_placeholder_ranges(num_items=num_images,
88
+ item_size=image_feature_size)
89
+ }
90
+ return DummyData(seq_data, mm_data, mm_placeholders)
91
92
93
def input_mapper_for_pixtral(ctx: InputContext,
@@ -630,13 +636,13 @@ def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
636
637
638
def dummy_seq_data_for_pixtral_hf(
633
- hf_config: PixtralVisionConfig,
634
- seq_len: int,
635
- num_images: int,
636
- *,
637
- image_token_id: int,
638
- image_feature_size_override: Optional[int] = None,
639
-):
639
+ hf_config: PixtralVisionConfig,
640
+ seq_len: int,
641
+ num_images: int,
642
+ *,
643
+ image_token_id: int,
644
+ image_feature_size_override: Optional[int] = None,
645
+ mm_key: str = "image"):
646
if image_feature_size_override is None:
647
image_feature_size = get_max_pixtral_hf_image_feature_size(hf_config)
648
else:
@@ -645,7 +651,11 @@ def dummy_seq_data_for_pixtral_hf(
651
return SequenceData.from_prompt_token_counts(
652
(image_token_id, image_feature_size * num_images),
653
(0, seq_len - image_feature_size * num_images),
648
- )
654
+ ), {
655
+ mm_key:
656
+ consecutive_placeholder_ranges(num_items=num_images,
657
+ item_size=image_feature_size)
658
+ }
659
660
661
def dummy_image_for_pixtral_hf(
vllm/model_executor/models/qwen.py
+5
-5
@@ -23,8 +23,8 @@ from vllm.attention import Attention, AttentionMetadata
23
from vllm.compilation.decorators import support_torch_compile
24
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
25
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
27
- token_inputs)
26
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
27
+ InputContext, token_inputs)
28
from vllm.logger import init_logger
29
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
30
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -810,7 +810,7 @@ def dummy_data_for_qwen(
810
ctx: InputContext,
811
seq_len: int,
812
mm_counts: Mapping[str, int],
813
-) -> Tuple[SequenceData, Optional[Dict]]:
813
+) -> DummyData:
814
"""Build dummy data for warming up Qwen models; this will only contain text
815
matching the defaults for VLLM unless the model has a visual config.
816
@@ -829,7 +829,7 @@ def dummy_data_for_qwen(
829
if not hasattr(hf_config, "visual"):
830
seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
831
mm_data = None
832
- return seq_data, mm_data
832
+ return DummyData(seq_data, mm_data)
833
834
# We have a visual component - use images to warm up
835
num_images = mm_counts["image"]
@@ -861,7 +861,7 @@ def dummy_data_for_qwen(
861
# the data will get resized and the # of tokens per image is constant
862
image = Image.new("RGB", (224, 224), color=0)
863
mm_data = {"image": image if num_images == 1 else [image] * num_images}
864
- return seq_data, mm_data
864
+ return DummyData(seq_data, mm_data)
865
866
867
class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
vllm/model_executor/models/qwen2_audio.py
+11
-4
@@ -31,8 +31,8 @@ from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
31
32
from vllm.attention import AttentionMetadata
33
from vllm.config import CacheConfig, MultiModalConfig
34
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
35
- token_inputs)
34
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35
+ InputContext, token_inputs)
36
from vllm.logger import init_logger
37
from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
@@ -44,6 +44,7 @@ from vllm.model_executor.model_loader.weight_utils import (
44
from vllm.model_executor.models.qwen2 import Qwen2Model
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
47
+from vllm.multimodal.utils import consecutive_placeholder_ranges
48
from vllm.sequence import IntermediateTensors, SequenceData
49
50
from .interfaces import SupportsMultiModal, SupportsPP
@@ -85,7 +86,8 @@ class Qwen2AudioMultiModalProjector(nn.Module):
86
def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
87
mm_counts: Mapping[str, int]):
88
num_audios = mm_counts["audio"]
88
- max_llm_audio_tokens = get_max_qwen2_audio_audio_tokens(ctx) * num_audios
89
+ max_tokens_per_audio = get_max_qwen2_audio_audio_tokens(ctx)
90
+ max_llm_audio_tokens = max_tokens_per_audio * num_audios
91
if seq_len - max_llm_audio_tokens - 2 < 0:
92
raise RuntimeError(
93
f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
@@ -99,7 +101,12 @@ def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
101
(0, seq_len - max_llm_audio_tokens),
102
)
103
dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
102
- return dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}
104
+ return DummyData(
105
+ dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}, {
106
+ "audio":
107
+ consecutive_placeholder_ranges(num_items=num_audios,
108
+ item_size=max_tokens_per_audio)
109
+ })
110
111
112
def get_processor(
vllm/model_executor/models/qwen2_vl.py
+6
-5
@@ -44,8 +44,8 @@ from vllm.attention.selector import _Backend
44
from vllm.config import CacheConfig, MultiModalConfig
45
from vllm.distributed import get_pp_group, parallel_state
46
from vllm.distributed import utils as dist_utils
47
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
48
- token_inputs)
47
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
48
+ InputContext, token_inputs)
49
from vllm.logger import init_logger
50
from vllm.model_executor import SamplingMetadata
51
from vllm.model_executor.layers.activation import QuickGELU
@@ -744,9 +744,10 @@ def dummy_data_for_qwen2_vl(
744
dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
745
color=0)
746
747
- return dummy_seqdata, {
748
- "image": dummy_image if num_images == 1 else [dummy_image] * num_images
749
- }
747
+ return DummyData(dummy_seqdata, {
748
+ "image":
749
+ dummy_image if num_images == 1 else [dummy_image] * num_images
750
+ })
751
752
753
def _get_llm_num_vision_tokens(
vllm/model_executor/models/siglip.py
+17
-7
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23
VocabParallelEmbedding)
24
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25
from vllm.multimodal.utils import (cached_get_tokenizer,
26
+ consecutive_placeholder_ranges,
27
repeat_and_pad_placeholder_tokens)
28
from vllm.sequence import SequenceData
29
@@ -61,6 +62,7 @@ def dummy_seq_data_for_siglip(
62
*,
63
image_token_id: int,
64
image_feature_size_override: Optional[int] = None,
65
+ mm_key: str = "image",
66
):
67
if image_feature_size_override is None:
68
image_feature_size = get_siglip_image_feature_size(hf_config)
@@ -70,7 +72,11 @@ def dummy_seq_data_for_siglip(
72
return SequenceData.from_prompt_token_counts(
73
(image_token_id, image_feature_size * num_images),
74
(0, seq_len - image_feature_size * num_images),
73
- )
75
+ ), {
76
+ mm_key:
77
+ consecutive_placeholder_ranges(num_items=num_images,
78
+ item_size=image_feature_size)
79
+ }
80
81
82
def dummy_image_for_siglip(
@@ -122,6 +128,11 @@ def input_processor_for_siglip(
128
if multi_modal_data is None or "image" not in multi_modal_data:
129
return inputs
130
131
+ if "multi_modal_placeholders" in inputs and "image" in inputs[
132
+ "multi_modal_placeholders"]:
133
+ # The inputs already have placeholders.
134
+ return inputs
135
+
136
tokenizer = cached_get_tokenizer(model_config.tokenizer)
137
138
if image_feature_size_override is None:
@@ -135,7 +146,7 @@ def input_processor_for_siglip(
146
else:
147
image_feature_size = image_feature_size_override
148
138
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
149
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
150
tokenizer,
151
inputs.get("prompt"),
152
inputs["prompt_token_ids"],
@@ -144,11 +155,10 @@ def input_processor_for_siglip(
155
)
156
157
# NOTE: Create a defensive copy of the original inputs
147
- return token_inputs(
148
- prompt_token_ids=new_token_ids,
149
- prompt=new_prompt,
150
- multi_modal_data=multi_modal_data,
151
- )
158
+ return token_inputs(prompt_token_ids=new_token_ids,
159
+ prompt=new_prompt,
160
+ multi_modal_data=multi_modal_data,
161
+ multi_modal_placeholders={"image": ranges})
162
163
164
# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
vllm/model_executor/models/ultravox.py
+32
-28
@@ -2,7 +2,6 @@
2
"""PyTorch Ultravox model."""
3
4
import math
5
-from array import array
5
from functools import cached_property, lru_cache
6
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
7
TypedDict, Union, cast)
@@ -17,27 +16,27 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
16
17
from vllm.attention import AttentionMetadata
18
from vllm.config import CacheConfig, MultiModalConfig
20
-from vllm.inputs import INPUT_REGISTRY
21
-from vllm.inputs.data import DecoderOnlyInputs, token_inputs
22
-from vllm.inputs.registry import InputContext
19
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20
+ InputContext, token_inputs)
21
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22
from vllm.model_executor.layers.layernorm import RMSNorm
23
from vllm.model_executor.layers.quantization import QuantizationConfig
24
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25
from vllm.model_executor.model_loader.loader import DefaultModelLoader
26
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
-from vllm.multimodal import MULTIMODAL_REGISTRY
30
-from vllm.multimodal.base import MultiModalInputs, NestedTensors
27
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
28
+ NestedTensors)
29
from vllm.multimodal.utils import (cached_get_tokenizer,
30
+ consecutive_placeholder_ranges,
31
repeat_and_pad_placeholder_tokens)
33
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
34
- SequenceData)
32
+from vllm.sequence import IntermediateTensors, SequenceData
33
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34
from vllm.utils import is_list_of
35
36
from .interfaces import SupportsMultiModal, SupportsPP
37
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
40
- init_vllm_registered_model, merge_multimodal_embeddings)
38
+ init_vllm_registered_model,
39
+ merge_multimodal_embeddings_from_map)
40
41
_AUDIO_PLACEHOLDER_TOKEN = 128002
42
_AUDIO_TOKENS_PER_SECOND = 6.25
@@ -46,13 +45,13 @@ _AUDIO_TOKENS_PER_SECOND = 6.25
45
class UltravoxAudioFeatureInputs(TypedDict):
46
type: Literal["audio_features"]
47
data: NestedTensors
49
- """Shape: `(batch_size, num_audios, 80, M)"""
48
+ """Shape: `(batch_size, num_audios, 80, M)`"""
49
50
51
class UltravoxAudioEmbeddingInputs(TypedDict):
52
type: Literal["audio_embeds"]
53
data: NestedTensors
55
- """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
54
+ """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)`"""
55
56
57
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
@@ -79,17 +78,16 @@ def dummy_seq_data_for_ultravox(
78
seq_len: int,
79
audio_count: int,
80
):
82
- audio_placeholder = array(
83
- VLLM_TOKEN_ID_ARRAY_TYPE,
84
- [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
81
+ audio_length = min(get_ultravox_max_audio_tokens(ctx),
82
+ seq_len // audio_count)
83
86
- # Add a separator between each chunk.
87
- audio_token_ids = (audio_placeholder +
88
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [0])) * audio_count
89
- other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
90
- [0]) * (seq_len - len(audio_token_ids))
91
-
92
- return SequenceData(audio_token_ids + other_token_ids)
84
+ return SequenceData.from_prompt_token_counts(
85
+ (_AUDIO_PLACEHOLDER_TOKEN, audio_length * audio_count),
86
+ (0, seq_len - audio_length * audio_count)), {
87
+ "audio":
88
+ consecutive_placeholder_ranges(num_items=audio_count,
89
+ item_size=audio_length)
90
+ }
91
92
93
def dummy_audio_for_ultravox(
@@ -107,10 +105,10 @@ def dummy_data_for_ultravox(
105
mm_counts: Mapping[str, int],
106
):
107
audio_count = mm_counts["audio"]
110
- seq_data = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
108
+ seq_data, ranges = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
109
mm_dict = dummy_audio_for_ultravox(ctx, audio_count)
110
113
- return (seq_data, mm_dict)
111
+ return DummyData(seq_data, mm_dict, ranges)
112
113
114
def input_mapper_for_ultravox(ctx: InputContext, data: object):
@@ -164,6 +162,11 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
162
if multi_modal_data is None or "audio" not in multi_modal_data:
163
return inputs
164
165
+ if "multi_modal_placeholders" in inputs and "audio" in inputs[
166
+ "multi_modal_placeholders"]:
167
+ # The inputs already have placeholders.
168
+ return inputs
169
+
170
feature_extractor = whisper_feature_extractor(ctx)
171
audios = multi_modal_data["audio"]
172
if not isinstance(audios, list):
@@ -197,7 +200,7 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
200
201
tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
202
200
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
203
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
204
tokenizer,
205
inputs.get("prompt"),
206
inputs["prompt_token_ids"],
@@ -208,7 +211,8 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
211
# NOTE: Create a defensive copy of the original inputs
212
return token_inputs(prompt_token_ids=new_token_ids,
213
prompt=new_prompt,
211
- multi_modal_data=multi_modal_data)
214
+ multi_modal_data=multi_modal_data,
215
+ multi_modal_placeholders={"audio": ranges})
216
217
218
class StackAudioFrames(nn.Module):
@@ -472,9 +476,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
476
inputs_embeds = self.language_model.model.get_input_embeddings(
477
input_ids)
478
475
- inputs_embeds = merge_multimodal_embeddings(
476
- input_ids, inputs_embeds, audio_embeddings,
477
- _AUDIO_PLACEHOLDER_TOKEN)
479
+ merge_multimodal_embeddings_from_map(
480
+ inputs_embeds, audio_embeddings,
481
+ attn_metadata.multi_modal_placeholder_index_maps["audio"])
482
input_ids = None
483
else:
484
inputs_embeds = None
vllm/model_executor/models/utils.py
+17
-1
@@ -18,7 +18,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.model_loader.loader import build_model
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.model_executor.models import ModelRegistry
21
-from vllm.multimodal.base import NestedTensors
21
+from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
22
from vllm.platforms import current_platform
23
from vllm.sequence import IntermediateTensors
24
from vllm.utils import is_pin_memory_available
@@ -326,6 +326,22 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
326
_embedding_count_expression(inner) for inner in embeddings)
327
328
329
+def merge_multimodal_embeddings_from_map(
330
+ inputs_embeds: torch.Tensor, multimodal_embeddings: NestedTensors,
331
+ placeholder_map: MultiModalPlaceholderMap.IndexMap) -> torch.Tensor:
332
+ """
333
+ Merge ``multimodal_embeddings`` into ``inputs_embeds`` using the provided
334
+ placeholder map .
335
+
336
+ Note:
337
+ This updates ``inputs_embeds`` in place.
338
+ """
339
+ flattened_embeddings = _flatten_embeddings(multimodal_embeddings)
340
+ inputs_embeds[placeholder_map.dest] = flattened_embeddings[
341
+ placeholder_map.src]
342
+ return inputs_embeds
343
+
344
+
345
def _merge_multimodal_embeddings(
346
inputs_embeds: torch.Tensor,
347
is_multimodal: torch.Tensor,
vllm/multimodal/__init__.py
+5
-2
@@ -1,6 +1,7 @@
1
from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2
- MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3
- NestedTensors)
2
+ MultiModalDataDict, MultiModalInputs,
3
+ MultiModalPlaceholderDict, MultiModalPlaceholderMap,
4
+ MultiModalPlugin, NestedTensors)
5
from .registry import MultiModalRegistry
6
7
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -17,6 +18,8 @@ __all__ = [
18
"MultiModalDataBuiltins",
19
"MultiModalDataDict",
20
"MultiModalInputs",
21
+ "MultiModalPlaceholderDict",
22
+ "MultiModalPlaceholderMap",
23
"MultiModalPlugin",
24
"NestedTensors",
25
"MULTIMODAL_REGISTRY",
vllm/multimodal/base.py
+209
-5
@@ -1,8 +1,9 @@
1
import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
-from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type,
5
- TypedDict, TypeVar, Union, cast, final)
4
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
5
+ NamedTuple, Optional, Tuple, Type, TypedDict, TypeVar,
6
+ Union, cast, final)
7
8
import numpy as np
9
import torch
@@ -11,12 +12,15 @@ from PIL import Image
12
from torch import nn
13
from typing_extensions import TypeAlias
14
14
-from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
17
from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18
json_map_leaves, resolve_mm_processor_kwargs)
19
20
+if TYPE_CHECKING:
21
+ from vllm.config import ModelConfig
22
+ from vllm.sequence import SequenceGroupMetadata
23
+
24
logger = init_logger(__name__)
25
26
NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
@@ -151,6 +155,30 @@ Note:
155
Read more on that :ref:`here <adding_multimodal_plugin>`.
156
"""
157
158
+
159
+class PlaceholderRange(TypedDict):
160
+ """
161
+ Placeholder location information for multi-modal data.
162
+
163
+ For example:
164
+ Prompt: AAAA BBBB What is in these images?
165
+ Images A and B will have:
166
+ A: { "offset": 0, "length": 4 }
167
+ B: { "offset": 5, "length": 4 }
168
+ """
169
+
170
+ offset: int
171
+ """The start index of the placeholder in the prompt."""
172
+
173
+ length: int
174
+ """The length of the placeholder."""
175
+
176
+
177
+MultiModalPlaceholderDict = Mapping[str, List[PlaceholderRange]]
178
+"""
179
+A dictionary containing placeholder ranges.
180
+"""
181
+
182
MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
183
MultiModalInputs]
184
"""
@@ -243,7 +271,7 @@ class MultiModalPlugin(ABC):
271
272
return wrapper
273
246
- def map_input(self, model_config: ModelConfig,
274
+ def map_input(self, model_config: "ModelConfig",
275
data: MultiModalData[object],
276
mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
277
"""
@@ -332,7 +360,7 @@ class MultiModalPlugin(ABC):
360
361
return wrapper
362
335
- def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
363
+ def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
364
"""
365
Get the maximum number of multi-modal tokens
366
for profiling the memory usage of a model.
@@ -366,3 +394,179 @@ class MultiModalPlugin(ABC):
394
self._validate_max_multimodal_tokens(max_mm_tokens)
395
396
return max_mm_tokens
397
+
398
+
399
+class MultiModalPlaceholderMap:
400
+ """
401
+ Relates multi-modal embeddings to their corresponding placeholders.
402
+ """
403
+
404
+ class IndexMap(NamedTuple):
405
+ src: List[int]
406
+ dest: List[int]
407
+
408
+ src_ranges: List[range]
409
+ """
410
+ The indices of the multi-modal embeddings that will replace the
411
+ corresponding placeholder embeddings pointed to by ``dest_ranges``.
412
+ """
413
+
414
+ src_len: int
415
+ """
416
+ The total number of flattened multi-modal embeddings.
417
+ """
418
+
419
+ dest_ranges: List[range]
420
+ """
421
+ The indices of the placeholder embeddings that will be replaced by the
422
+ multimodal embeddings.
423
+ """
424
+
425
+ dest_len: int
426
+ """
427
+ The total number of embeddings in the destination tensor.
428
+ """
429
+
430
+ def __init__(self):
431
+ self.src_ranges = []
432
+ self.src_len = 0
433
+ self.dest_ranges = []
434
+ self.dest_len = 0
435
+
436
+ @classmethod
437
+ def from_seq_group(
438
+ cls, seq_group: "SequenceGroupMetadata", positions: range
439
+ ) -> Tuple[Optional[MultiModalDataDict], Dict[str,
440
+ "MultiModalPlaceholderMap"]]:
441
+ """
442
+ Returns the multi-modal items that intersect with the portion of a
443
+ prompt (``seq_group``) represented by ``positions``, as well as a
444
+ ``MultiModalPlaceholderMap`` that relates the multi-modal embedding
445
+ vectors to their corresponding placeholders.
446
+
447
+ Consider the following scenarios:
448
+
449
+ Prompt: |AAAA BBBB What's in these images?|
450
+ Positions: |.................................|
451
+
452
+ images = [A, B]
453
+ src_ranges = [(0, 4), (4, 8)]
454
+ dest_ranges = [(0, 4), (5, 9)]
455
+
456
+ Prompt: |AAAA BBBB What's in these images?|
457
+ Positions: | ..... |
458
+
459
+ images = [A, B]
460
+ src_ranges = [(2, 4), (4, 6)]
461
+ dest_ranges = [(0, 2), (3, 5)]
462
+
463
+ Prompt: |AAAA BBBB What's in these images?|
464
+ Positions: | ......... |
465
+
466
+ images = [B]
467
+ src_ranges = [(0, 4)]
468
+ dest_ranges = [(0, 4)]
469
+
470
+ Prompt: |AAAA BBBB What's in these images?|
471
+ Positions: | .......................|
472
+
473
+ images = []
474
+ src_ranges = []
475
+ dest_ranges = []
476
+ """
477
+ if (not seq_group.multi_modal_data
478
+ or not seq_group.multi_modal_placeholders):
479
+ return seq_group.multi_modal_data, {}
480
+
481
+ mm_data = {**seq_group.multi_modal_data}
482
+ placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
483
+ MultiModalPlaceholderMap)
484
+
485
+ for modality, placeholders in seq_group.multi_modal_placeholders.items(
486
+ ):
487
+ mm_items = mm_data.pop(modality)
488
+ if not isinstance(mm_items, list):
489
+ mm_items = [mm_items]
490
+
491
+ if positions:
492
+ intersecting_items = placeholder_maps[
493
+ modality].append_items_from_seq_group(
494
+ positions, mm_items, placeholders)
495
+
496
+ if intersecting_items:
497
+ mm_data[modality] = intersecting_items
498
+
499
+ return mm_data, placeholder_maps
500
+
501
+ def append_items_from_seq_group(
502
+ self, positions: range, multi_modal_items: List[_T],
503
+ multi_modal_placeholders: List[PlaceholderRange]) -> List[_T]:
504
+ """
505
+ Adds the multi-modal items that intersect ```positions`` to this
506
+ placeholder map and returns the intersecting items.
507
+ """
508
+ intersecting_items = []
509
+
510
+ if len(multi_modal_items) != len(multi_modal_placeholders):
511
+ raise ValueError(
512
+ "Multi-modal placeholders and items must have the same length."
513
+ )
514
+ for placeholder_dict, mm_item in zip(multi_modal_placeholders,
515
+ multi_modal_items):
516
+ placeholder = range(
517
+ placeholder_dict["offset"],
518
+ placeholder_dict["offset"] + placeholder_dict["length"])
519
+ intersection = range(max(positions.start, placeholder.start),
520
+ min(positions.stop, placeholder.stop))
521
+
522
+ if not intersection:
523
+ # Skip this multi-modal item.
524
+ continue
525
+
526
+ token_embedding_range = range(intersection.start - positions.start,
527
+ intersection.stop - positions.start)
528
+
529
+ multimodal_embedding_range = range(
530
+ intersection.start - placeholder.start + self.src_len,
531
+ intersection.stop - placeholder.start + self.src_len)
532
+
533
+ intersecting_items.append(mm_item)
534
+ self.dest_ranges.append(token_embedding_range)
535
+ self.src_ranges.append(multimodal_embedding_range)
536
+ self.src_len += len(placeholder)
537
+
538
+ self.dest_len += len(positions)
539
+ return intersecting_items
540
+
541
+ def extend(self, other: "MultiModalPlaceholderMap"):
542
+ """
543
+ Adds the placeholders from another ``MultiModalPlaceholderMap`` to this
544
+ instance based on the source and destination tensors being
545
+ concatenated.
546
+ """
547
+
548
+ self.src_ranges.extend(
549
+ range(self.src_len + r.start, self.src_len + r.stop)
550
+ for r in other.src_ranges)
551
+ self.src_len += other.src_len
552
+ self.dest_ranges.extend(
553
+ range(self.dest_len + r.start, self.dest_len + r.stop)
554
+ for r in other.dest_ranges)
555
+ self.dest_len += other.dest_len
556
+
557
+ def index_map(self) -> "IndexMap":
558
+ """
559
+ Finalizes the placeholder map into lists of indices that can be used to
560
+ index the source and destination tensors.
561
+ """
562
+
563
+ src_indices = [i for r in self.src_ranges for i in r]
564
+ dest_indices = [i for r in self.dest_ranges for i in r]
565
+
566
+ if len(src_indices) != len(dest_indices):
567
+ raise ValueError(
568
+ f"The number of source ({len(src_indices)}) and destination "
569
+ f"indices ({len(dest_indices)}) must be the same.")
570
+
571
+ return MultiModalPlaceholderMap.IndexMap(src=src_indices,
572
+ dest=dest_indices)
vllm/multimodal/image.py
+5
-3
@@ -1,11 +1,10 @@
1
from functools import lru_cache
2
-from typing import Any, Dict, Optional
2
+from typing import TYPE_CHECKING, Any, Dict, Optional
3
4
import torch
5
from PIL import Image
6
from transformers.image_processing_base import BatchFeature
7
8
-from vllm.config import ModelConfig
8
from vllm.inputs.registry import InputContext
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.processor import get_image_processor
@@ -13,6 +12,9 @@ from vllm.utils import is_list_of
12
13
from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
14
15
+if TYPE_CHECKING:
16
+ from vllm.config import ModelConfig
17
+
18
logger = init_logger(__name__)
19
20
cached_get_image_processor = lru_cache(get_image_processor)
@@ -26,7 +28,7 @@ class ImagePlugin(MultiModalPlugin):
28
29
def _get_hf_image_processor(
30
self,
29
- model_config: ModelConfig,
31
+ model_config: "ModelConfig",
32
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
33
):
34
if mm_processor_kwargs is None:
vllm/multimodal/registry.py
+10
-8
@@ -1,8 +1,7 @@
1
import functools
2
from collections import UserDict
3
-from typing import Any, Dict, Mapping, Optional, Sequence
3
+from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Sequence
4
5
-from vllm.config import ModelConfig
5
from vllm.logger import init_logger
6
7
from .audio import AudioPlugin
@@ -11,6 +10,9 @@ from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10
from .image import ImagePlugin
11
from .video import VideoPlugin
12
13
+if TYPE_CHECKING:
14
+ from vllm.config import ModelConfig
15
+
16
logger = init_logger(__name__)
17
18
@@ -20,7 +22,7 @@ class _MultiModalLimits(UserDict):
22
when attempting to access a model that does not exist.
23
"""
24
23
- def __getitem__(self, key: ModelConfig) -> Dict[str, int]:
25
+ def __getitem__(self, key: "ModelConfig") -> Dict[str, int]:
26
try:
27
return super().__getitem__(key)
28
except KeyError as exc:
@@ -98,7 +100,7 @@ class MultiModalRegistry:
100
101
def map_input(
102
self,
101
- model_config: ModelConfig,
103
+ model_config: "ModelConfig",
104
data: MultiModalDataDict,
105
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
106
) -> MultiModalInputs:
@@ -139,7 +141,7 @@ class MultiModalRegistry:
141
142
return MultiModalInputs(merged_dict)
143
142
- def create_input_mapper(self, model_config: ModelConfig):
144
+ def create_input_mapper(self, model_config: "ModelConfig"):
145
"""
146
Create an input mapper (see :meth:`map_input`) for a specific model.
147
"""
@@ -177,7 +179,7 @@ class MultiModalRegistry:
179
"""
180
return self.register_max_multimodal_tokens("image", max_mm_tokens)
181
180
- def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
182
+ def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
183
"""
184
Get the maximum number of multi-modal tokens
185
for profiling the memory usage of a model.
@@ -195,7 +197,7 @@ class MultiModalRegistry:
197
198
def init_mm_limits_per_prompt(
199
self,
198
- model_config: ModelConfig,
200
+ model_config: "ModelConfig",
201
) -> None:
202
"""
203
Initialize the maximum number of multi-modal input instances for each
@@ -231,7 +233,7 @@ class MultiModalRegistry:
233
234
def get_mm_limits_per_prompt(
235
self,
234
- model_config: ModelConfig,
236
+ model_config: "ModelConfig",
237
) -> Mapping[str, int]:
238
"""
239
Get the maximum number of multi-modal input instances for each modality
vllm/multimodal/utils.py
+18
-3
@@ -10,7 +10,7 @@ from PIL import Image
10
from vllm.connections import global_http_connection
11
from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
12
from vllm.logger import init_logger
13
-from vllm.multimodal.base import MultiModalDataDict
13
+from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
14
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
15
16
logger = init_logger(__name__)
@@ -258,7 +258,7 @@ def repeat_and_pad_placeholder_tokens(
258
repeat_count: Union[int, List[int]],
259
pad_token_left: Optional[int] = None,
260
pad_token_right: Optional[int] = None,
261
-) -> Tuple[Optional[str], List[int]]:
261
+) -> Tuple[Optional[str], List[int], List[PlaceholderRange]]:
262
if isinstance(repeat_count, int):
263
repeat_count = [repeat_count]
264
@@ -301,6 +301,7 @@ def repeat_and_pad_placeholder_tokens(
301
new_prompt += prompt_parts[-1]
302
303
new_token_ids: List[int] = []
304
+ placeholder_ranges: List[PlaceholderRange] = []
305
placeholder_token_idx = 0
306
for i, token in enumerate(prompt_token_ids):
307
if token == placeholder_token_id:
@@ -310,6 +311,10 @@ def repeat_and_pad_placeholder_tokens(
311
pad_token_left=pad_token_left,
312
pad_token_right=pad_token_right,
313
)
314
+ placeholder_ranges.append({
315
+ "offset": len(new_token_ids),
316
+ "length": len(replacement_ids)
317
+ })
318
new_token_ids.extend(replacement_ids)
319
placeholder_token_idx += 1
320
@@ -320,4 +325,14 @@ def repeat_and_pad_placeholder_tokens(
325
else:
326
new_token_ids.append(token)
327
323
- return new_prompt, new_token_ids
328
+ return new_prompt, new_token_ids, placeholder_ranges
329
+
330
+
331
+def consecutive_placeholder_ranges(num_items: int,
332
+ item_size: int) -> List[PlaceholderRange]:
333
+ """Returns a list of consecutive PlaceholderRanges of a fixed size"""
334
+
335
+ return [
336
+ PlaceholderRange(offset=i * item_size, length=item_size)
337
+ for i in range(num_items)
338
+ ]
vllm/multimodal/video.py
+9
-5
@@ -1,18 +1,19 @@
1
from functools import lru_cache
2
-from typing import Any, Dict, List, Optional, Union
2
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
3
4
import numpy as np
5
6
-from vllm.config import ModelConfig
6
from vllm.inputs.registry import InputContext
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.processor import get_video_processor
9
from vllm.transformers_utils.tokenizer import get_tokenizer
11
-from vllm.utils import is_list_of
10
11
from .base import MultiModalData, MultiModalInputs
12
from .image import ImagePlugin
13
14
+if TYPE_CHECKING:
15
+ from vllm.config import ModelConfig
16
+
17
logger = init_logger(__name__)
18
19
cached_get_video_processor = lru_cache(get_video_processor)
@@ -38,7 +39,7 @@ class VideoPlugin(ImagePlugin):
39
40
def _get_hf_video_processor(
41
self,
41
- model_config: ModelConfig,
42
+ model_config: "ModelConfig",
43
mm_processor_kwargs: Optional[Dict[str, Any]] = None,
44
):
45
if mm_processor_kwargs is None:
@@ -56,7 +57,10 @@ class VideoPlugin(ImagePlugin):
57
) -> MultiModalInputs:
58
model_config = ctx.model_config
59
59
- if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
60
+ if isinstance(data, list) and len(data) == 1:
61
+ data = data[0]
62
+
63
+ if isinstance(data, np.ndarray):
64
video_processor = self._get_hf_video_processor(
65
model_config,
66
mm_processor_kwargs,
vllm/sequence.py
+13
-4
@@ -15,13 +15,13 @@ import torch
15
16
from vllm.inputs.parse import is_encoder_decoder_inputs
17
from vllm.lora.request import LoRARequest
18
+from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
19
from vllm.pooling_params import PoolingParams
20
from vllm.prompt_adapter.request import PromptAdapterRequest
21
from vllm.sampling_params import RequestOutputKind, SamplingParams
22
23
if TYPE_CHECKING:
24
from vllm.inputs import SingletonInputs
24
- from vllm.multimodal.base import MultiModalDataDict
25
26
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
27
@@ -485,7 +485,7 @@ class Sequence:
485
return cast(List[int], self.inputs.get(prompt_token_ids_key))
486
487
@property
488
- def multi_modal_data(self) -> "MultiModalDataDict":
488
+ def multi_modal_data(self) -> MultiModalDataDict:
489
inputs = self.inputs
490
491
if (inputs.get("multi_modal_data")
@@ -495,11 +495,15 @@ class Sequence:
495
)
496
497
return cast(
498
- "MultiModalDataDict",
498
+ MultiModalDataDict,
499
(inputs.get("multi_modal_data")
500
or inputs.get("encoder_multi_modal_data") or {}),
501
)
502
503
+ @property
504
+ def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
505
+ return self.inputs.get("multi_modal_placeholders") or {}
506
+
507
@property
508
def mm_processor_kwargs(self) -> Dict[str, Any]:
509
return self.inputs.get("mm_processor_kwargs") or {}
@@ -728,9 +732,13 @@ class SequenceGroup:
732
if self.encoder_seq is not None else None)
733
734
@property
731
- def multi_modal_data(self) -> "MultiModalDataDict":
735
+ def multi_modal_data(self) -> MultiModalDataDict:
736
return self.first_seq.multi_modal_data
737
738
+ @property
739
+ def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
740
+ return self.first_seq.multi_modal_placeholders
741
+
742
@property
743
def mm_processor_kwargs(self) -> Dict[str, Any]:
744
return self.first_seq.mm_processor_kwargs
@@ -946,6 +954,7 @@ class SequenceGroupMetadata(
954
# "MultiModalDataDict" types. We have to use Any due to msgspec
955
# doesn't allow to have union of 2 different dicts.
956
multi_modal_data: Optional[Any] = None
957
+ multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
958
mm_processor_kwargs: Optional[Dict[str, Any]] = None
959
encoder_seq_data: Optional[SequenceData] = None
960
cross_block_table: Optional[List[int]] = None
vllm/worker/cpu_model_runner.py
+31
-7
@@ -1,5 +1,6 @@
1
import dataclasses
2
import weakref
3
+from collections import defaultdict
4
from dataclasses import dataclass
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
6
@@ -16,7 +17,7 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
17
from vllm.model_executor.layers.sampler import SamplerOutput
18
from vllm.model_executor.model_loader import get_model
19
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
- MultiModalInputs)
20
+ MultiModalInputs, MultiModalPlaceholderMap)
21
from vllm.sequence import (IntermediateTensors, SequenceData,
22
SequenceGroupMetadata)
23
from vllm.transformers_utils.config import uses_mrope
@@ -148,9 +149,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
149
query_lens=seq_lens,
150
)
151
151
- def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
152
- computed_len: int,
152
+ def _compute_multi_modal_input(self, seq_group: SequenceGroupMetadata,
153
+ seq_data: SequenceData, computed_len: int,
154
mm_processor_kwargs: Dict[str, Any]):
155
+
156
+ # NOTE: mm_data only includes the subset of multi-modal items that
157
+ # intersect with the current prefill positions.
158
+ mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
159
+ seq_group, range(computed_len, len(seq_data.get_token_ids())))
160
+
161
+ if not mm_data:
162
+ return
163
+
164
mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
165
166
# special processing for mrope position deltas.
@@ -179,7 +189,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
189
context_len=computed_len,
190
)
191
seq_data.mrope_position_delta = mrope_position_delta
182
- return mm_kwargs, mrope_positions
192
+ return mm_kwargs, placeholder_maps, mrope_positions
193
194
def _prepare_prompt(
195
self,
@@ -194,6 +204,9 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
204
slot_mapping: List[int] = []
205
seq_lens: List[int] = []
206
multi_modal_inputs_list: List[MultiModalInputs] = []
207
+ multi_modal_placeholder_maps: Dict[
208
+ str,
209
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
210
211
for seq_group_metadata in seq_group_metadata_list:
212
assert seq_group_metadata.is_prompt
@@ -210,11 +223,15 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
223
input_tokens.extend(prompt_tokens) # Token ids
224
225
mrope_positions = None
213
- if (mm_data := seq_group_metadata.multi_modal_data):
214
- mm_kwargs, mrope_positions = self._compute_multi_modal_input(
215
- seq_data, mm_data, computed_len,
226
+ if seq_group_metadata.multi_modal_data:
227
+ mm_kwargs, placeholder_maps, mrope_positions = self \
228
+ ._compute_multi_modal_input(
229
+ seq_group_metadata, seq_data, computed_len,
230
seq_group_metadata.mm_processor_kwargs)
231
multi_modal_inputs_list.append(mm_kwargs)
232
+ for modality, placeholder_map in placeholder_maps.items():
233
+ multi_modal_placeholder_maps[modality].extend(
234
+ placeholder_map)
235
236
# Token position ids
237
# NOTE(woosuk): Here we assume that the first token in the prompt
@@ -264,6 +281,11 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
281
slot_mapping = torch.tensor(slot_mapping,
282
dtype=torch.long,
283
device=self.device) # type: ignore
284
+ placeholder_index_maps = {
285
+ modality: placeholder_map.index_map()
286
+ for modality, placeholder_map in
287
+ multi_modal_placeholder_maps.items()
288
+ }
289
290
attn_metadata = self.attn_backend.make_metadata(
291
is_prompt=True,
@@ -275,6 +297,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
297
num_decode_tokens=0,
298
block_tables=torch.tensor([]),
299
slot_mapping=slot_mapping,
300
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
301
)
302
303
multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
@@ -366,6 +389,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
389
attn_metadata = self.attn_backend.make_metadata(
390
is_prompt=False,
391
slot_mapping=slot_mapping,
392
+ multi_modal_placeholder_index_maps=None,
393
seq_lens=seq_lens,
394
seq_lens_tensor=seq_lens_tensor,
395
max_decode_seq_len=max_decode_seq_len,
vllm/worker/enc_dec_model_runner.py
+17
-13
@@ -306,13 +306,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
306
(group_id < max_num_batched_tokens % max_num_seqs))
307
batch_size += seq_len
308
309
- decoder_seq_data, decoder_dummy_multi_modal_data \
310
- = self.input_registry.dummy_data_for_profiling(
311
- self.model_config,
309
+ decoder_dummy_data = self.input_registry \
310
+ .dummy_data_for_profiling(self.model_config,
311
seq_len,
312
self.mm_registry,
313
is_encoder_data=False)
315
- encoder_seq_data, encoder_dummy_multi_modal_data \
314
+ encoder_dummy_data \
315
= self.input_registry.dummy_data_for_profiling(
316
self.model_config,
317
seq_len,
@@ -320,26 +319,31 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
319
is_encoder_data=True)
320
321
# Having more tokens is over-conservative but otherwise fine
323
- assert len(decoder_seq_data.prompt_token_ids) >= seq_len, (
322
+ assert len(
323
+ decoder_dummy_data.seq_data.prompt_token_ids
324
+ ) >= seq_len, (
325
f"Expected at least {seq_len} dummy tokens for profiling, "
325
- f"but got: {len(decoder_seq_data.prompt_token_ids)}")
326
+ f"but got: {len(decoder_dummy_data.seq_data.prompt_token_ids)}"
327
+ )
328
327
- assert decoder_dummy_multi_modal_data is None or \
328
- encoder_dummy_multi_modal_data is None, (
329
+ assert decoder_dummy_data.multi_modal_data is None or \
330
+ encoder_dummy_data.multi_modal_data is None, (
331
"Multi-modal data can't be provided in both encoder and decoder"
332
)
333
334
seq = SequenceGroupMetadata(
335
request_id=str(group_id),
336
is_prompt=True,
335
- seq_data={group_id: decoder_seq_data},
337
+ seq_data={group_id: decoder_dummy_data.seq_data},
338
sampling_params=sampling_params,
339
block_tables=None,
338
- encoder_seq_data=encoder_seq_data,
340
+ encoder_seq_data=encoder_dummy_data.seq_data,
341
cross_block_table=None,
340
- multi_modal_data=decoder_dummy_multi_modal_data
341
- or encoder_dummy_multi_modal_data,
342
- )
342
+ multi_modal_data=decoder_dummy_data.multi_modal_data
343
+ or encoder_dummy_data.multi_modal_data,
344
+ multi_modal_placeholders=decoder_dummy_data.
345
+ multi_modal_placeholders
346
+ or encoder_dummy_data.multi_modal_placeholders)
347
seqs.append(seq)
348
349
# Run the model with the dummy inputs.
vllm/worker/model_runner.py
+16
-5
@@ -40,7 +40,8 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40
from vllm.model_executor.models import supports_lora, supports_multimodal
41
from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
42
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
43
- MultiModalInputs, MultiModalRegistry)
43
+ MultiModalInputs, MultiModalPlaceholderMap,
44
+ MultiModalRegistry)
45
from vllm.platforms import current_platform
46
from vllm.prompt_adapter.layers import PromptAdapterMapping
47
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -242,6 +243,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
243
244
# Multi-modal inputs.
245
multi_modal_inputs: Optional[MultiModalInputs] = None,
246
+ multi_modal_placeholder_maps: Optional[Dict[
247
+ str, MultiModalPlaceholderMap]] = None,
248
249
# Whether the prefix cache is hit (prefill only).
250
prefix_cache_hit: bool = False,
@@ -361,6 +364,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
364
365
self.prompt_adapter_request = prompt_adapter_request
366
self.multi_modal_inputs = multi_modal_inputs
367
+ self.multi_modal_placeholder_maps = multi_modal_placeholder_maps
368
self.prefix_cache_hit = prefix_cache_hit
369
370
self.n_seqs = len(self.seq_ids)
@@ -635,7 +639,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
639
def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
640
seq_group_metadata: SequenceGroupMetadata):
641
"""If multi-modal data is given, add it to the input."""
638
- mm_data = seq_group_metadata.multi_modal_data
642
+ # NOTE: mm_data only includes the subset of multi-modal items that
643
+ # intersect with the current prefill positions.
644
+ positions = inter_data.input_positions[0]
645
+ mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
646
+ seq_group_metadata,
647
+ range(positions[0], positions[0] + len(positions)))
648
if not mm_data:
649
return
650
@@ -643,6 +652,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
652
mm_data,
653
mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
654
inter_data.multi_modal_inputs = mm_kwargs
655
+ inter_data.multi_modal_placeholder_maps = placeholder_maps
656
657
# special processing for mrope position deltas.
658
if self.runner.model_is_mrope:
@@ -1255,7 +1265,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1265
(group_id < max_num_batched_tokens % max_num_seqs))
1266
batch_size += seq_len
1267
1258
- seq_data, dummy_multi_modal_data = self.input_registry \
1268
+ dummy_data = self.input_registry \
1269
.dummy_data_for_profiling(self.model_config,
1270
seq_len,
1271
self.mm_registry)
@@ -1263,12 +1273,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1273
seq = SequenceGroupMetadata(
1274
request_id=str(group_id),
1275
is_prompt=True,
1266
- seq_data={group_id: seq_data},
1276
+ seq_data={group_id: dummy_data.seq_data},
1277
sampling_params=sampling_params,
1278
block_tables=None,
1279
lora_request=dummy_lora_requests_per_seq[group_id]
1280
if dummy_lora_requests_per_seq else None,
1271
- multi_modal_data=dummy_multi_modal_data,
1281
+ multi_modal_data=dummy_data.multi_modal_data,
1282
+ multi_modal_placeholders=dummy_data.multi_modal_placeholders,
1283
)
1284
seqs.append(seq)
1285
vllm/worker/model_runner_base.py
+2
-3
@@ -46,9 +46,8 @@ def _init_attn_metadata_from_tensor_dict(
46
# Extract the fields used to create AttentionMetadata.
47
valid_attn_kwargs = {}
48
for field in dataclasses.fields(attn_backend.get_metadata_cls()):
49
- val = tensor_dict.pop(field.name, None)
50
- if val is not None:
51
- valid_attn_kwargs[field.name] = val
49
+ if field.name in tensor_dict:
50
+ valid_attn_kwargs[field.name] = tensor_dict.pop(field.name)
51
52
attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs)
53
tensor_dict["attn_metadata"] = attn_metadata
vllm/worker/openvino_model_runner.py
+31
-12
@@ -1,4 +1,5 @@
1
-from typing import List, NamedTuple, Optional, Tuple
1
+from collections import defaultdict
2
+from typing import Dict, List, NamedTuple, Optional, Tuple
3
4
import openvino as ov
5
import torch
@@ -14,7 +15,7 @@ from vllm.model_executor import SamplingMetadata
15
from vllm.model_executor.layers.sampler import SamplerOutput
16
from vllm.model_executor.model_loader.openvino import get_model
17
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17
- MultiModalInputs)
18
+ MultiModalInputs, MultiModalPlaceholderMap)
19
from vllm.sequence import SequenceGroupMetadata
20
21
logger = init_logger(__name__)
@@ -115,6 +116,9 @@ class OpenVINOModelRunner:
116
past_lens: List[int] = []
117
query_lens: List[int] = []
118
multi_modal_inputs_list: List[MultiModalInputs] = []
119
+ multi_modal_placeholder_maps: Dict[
120
+ str,
121
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
122
123
subsequence_begins: List[int] = []
124
block_indices: List[int] = []
@@ -168,15 +172,6 @@ class OpenVINOModelRunner:
172
and self.sliding_window is None
173
and is_prompt)
174
171
- mm_data = seq_group_metadata.multi_modal_data
172
- if mm_data:
173
- mm_kwargs = self.multi_modal_input_mapper(
174
- mm_data,
175
- mm_processor_kwargs=seq_group_metadata.
176
- mm_processor_kwargs,
177
- )
178
- multi_modal_inputs_list.append(mm_kwargs)
179
-
175
block_table = seq_group_metadata.block_tables[seq_id]
176
# TODO(sang): Combine chunked prefill and prefix caching by
177
# only allowing multiple of block_size chunk size.
@@ -220,7 +215,8 @@ class OpenVINOModelRunner:
215
query_lens.append(query_len)
216
217
input_tokens.extend(tokens)
223
- input_positions.extend(list(range(computed_len, seq_len)))
218
+ positions_range = range(computed_len, seq_len)
219
+ input_positions.extend(list(positions_range))
220
221
past_lens.append(computed_len)
222
subsequence_begins.append(subsequence_begins[-1] + query_len)
@@ -233,6 +229,22 @@ class OpenVINOModelRunner:
229
), "seq_len: {}, computed_len: {}, query_len: {}".format(
230
seq_len, computed_len, query_len)
231
232
+ if seq_group_metadata.multi_modal_data:
233
+ # NOTE: mm_data only includes the subset of multi-modal
234
+ # items that intersect with the current prefill positions.
235
+ mm_data, placeholder_maps = MultiModalPlaceholderMap \
236
+ .from_seq_group(seq_group_metadata, positions_range)
237
+
238
+ mm_kwargs = self.multi_modal_input_mapper(
239
+ mm_data,
240
+ mm_processor_kwargs=seq_group_metadata.
241
+ mm_processor_kwargs)
242
+ multi_modal_inputs_list.append(mm_kwargs)
243
+
244
+ for modality, placeholder_map in placeholder_maps.items():
245
+ multi_modal_placeholder_maps[modality].extend(
246
+ placeholder_map, )
247
+
248
max_query_len = max(query_lens)
249
assert max_query_len > 0, "query_lens: {}".format(query_lens)
250
@@ -261,12 +273,19 @@ class OpenVINOModelRunner:
273
max_context_len, dtype=torch.int32,
274
device=self.device) # type: ignore
275
276
+ placeholder_index_maps = {
277
+ modality: placeholder_map.index_map()
278
+ for modality, placeholder_map in
279
+ multi_modal_placeholder_maps.items()
280
+ }
281
+
282
attn_metadata = self.attn_backend.make_openvino_metadata(
283
past_lens=past_lens_tensor,
284
subsequence_begins=subsequence_begins_tensor,
285
block_indices=block_indices_tensor,
286
block_indices_begins=block_indices_begins_tensor,
287
max_context_len=max_context_len_tensor,
288
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
289
)
290
291
multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
vllm/worker/tpu_model_runner.py
+4
@@ -184,6 +184,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
184
num_prefill_tokens=batch_size * seq_len,
185
num_decode_tokens=0,
186
slot_mapping=slot_mapping,
187
+ multi_modal_placeholder_index_maps=None,
188
block_tables=None,
189
context_lens=None,
190
)
@@ -216,6 +217,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
217
num_prefill_tokens=0,
218
num_decode_tokens=batch_size * seq_len,
219
slot_mapping=slot_mapping,
220
+ multi_modal_placeholder_index_maps=None,
221
block_tables=block_tables,
222
context_lens=context_lens,
223
)
@@ -360,6 +362,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
362
num_prefill_tokens=0, # NOTE: This is not used.
363
num_decode_tokens=0,
364
slot_mapping=slot_mapping,
365
+ multi_modal_placeholder_index_maps=None,
366
block_tables=None,
367
context_lens=None,
368
)
@@ -429,6 +432,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
432
num_prefill_tokens=0,
433
num_decode_tokens=batch_size,
434
slot_mapping=slot_mapping,
435
+ multi_modal_placeholder_index_maps=None,
436
block_tables=block_tables,
437
context_lens=context_lens,
438
)
vllm/worker/xpu_model_runner.py
+32
-6
@@ -1,6 +1,7 @@
1
import dataclasses
2
import time
3
import weakref
4
+from collections import defaultdict
5
from dataclasses import dataclass
6
from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
7
Type, TypeVar)
@@ -19,7 +20,8 @@ from vllm.model_executor import SamplingMetadataCache
20
from vllm.model_executor.layers.sampler import SamplerOutput
21
from vllm.model_executor.model_loader import get_model
22
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
22
- MultiModalInputs, MultiModalRegistry)
23
+ MultiModalInputs, MultiModalPlaceholderMap,
24
+ MultiModalRegistry)
25
from vllm.sampling_params import SamplingParams
26
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
27
from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad
@@ -161,6 +163,9 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
163
slot_mapping: List[int] = []
164
seq_lens: List[int] = []
165
multi_modal_inputs_list: List[MultiModalInputs] = []
166
+ multi_modal_placeholder_maps: Dict[
167
+ str,
168
+ MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
169
170
for seq_group_metadata in seq_group_metadata_list:
171
assert seq_group_metadata.is_prompt
@@ -179,7 +184,21 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
184
# Token position ids
185
# NOTE(woosuk): Here we assume that the first token in the prompt
186
# is always the first token in the sequence.
182
- input_positions.extend(list(range(computed_len, seq_len)))
187
+ positions_range = range(computed_len, seq_len)
188
+ input_positions.extend(list(positions_range))
189
+
190
+ if seq_group_metadata.multi_modal_data:
191
+ # NOTE: mm_data only includes the subset of multi-modal items
192
+ # that intersect with the current prefill positions.
193
+ mm_data, placeholder_maps = MultiModalPlaceholderMap \
194
+ .from_seq_group(seq_group_metadata, positions_range)
195
+
196
+ mm_kwargs = self.runner.multi_modal_input_mapper(mm_data)
197
+ multi_modal_inputs_list.append(mm_kwargs)
198
+
199
+ for modality, placeholder_map in placeholder_maps.items():
200
+ multi_modal_placeholder_maps[modality].extend(
201
+ placeholder_map)
202
203
if seq_group_metadata.block_tables is None:
204
# During memory profiling, the block tables are not initialized
@@ -220,6 +239,11 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
239
slot_mapping = torch.tensor(slot_mapping,
240
dtype=torch.long,
241
device=self.device) # type: ignore
242
+ placeholder_index_maps = {
243
+ modality: placeholder_map.index_map()
244
+ for modality, placeholder_map in
245
+ multi_modal_placeholder_maps.items()
246
+ }
247
248
max_seqlen = max(seq_lens)
249
tmp = [0]
@@ -230,6 +254,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
254
attn_metadata = self.attn_backend.make_metadata(
255
is_prompt=True,
256
slot_mapping=slot_mapping,
257
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
258
seq_lens=seq_lens,
259
seqlen_q=seqlen_q,
260
max_seqlen=max_seqlen,
@@ -313,6 +338,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
338
attn_metadata = self.attn_backend.make_metadata(
339
is_prompt=False,
340
slot_mapping=slot_mapping,
341
+ multi_modal_placeholder_index_maps=None,
342
seq_lens=seq_lens,
343
seqlen_q=torch.tensor([]),
344
max_seqlen=0,
@@ -450,7 +476,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
476
(group_id < max_num_batched_tokens % max_num_seqs))
477
batch_size += seq_len
478
453
- seq_data, dummy_multi_modal_data = self.input_registry \
479
+ dummy_data = self.input_registry \
480
.dummy_data_for_profiling(self.model_config,
481
seq_len,
482
self.mm_registry)
@@ -458,12 +484,12 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
484
seq = SequenceGroupMetadata(
485
request_id=str(group_id),
486
is_prompt=True,
461
- seq_data={group_id: seq_data},
487
+ seq_data={group_id: dummy_data.seq_data},
488
sampling_params=sampling_params,
489
block_tables=None,
490
lora_request=None,
465
- multi_modal_data=dummy_multi_modal_data,
466
- )
491
+ multi_modal_data=dummy_data.multi_modal_data,
492
+ multi_modal_placeholders=dummy_data.multi_modal_placeholders)
493
seqs.append(seq)
494
495
# Run the model with the dummy inputs.