@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c0b7f548d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Core][VLM] Add precise multi-modal placeholder tracking (#8346)

Signed-off-by: Peter Salas <peter@fixie.ai>

Peter Salas committed Nov 1, 2024 at 16:21 UTC 6c0b7f548d80b5f61bfa472ad1497597c922dbc2
53 files changed +914 -282
examples/offline_inference_audio_language.py
+1 -5
@@ -34,11 +34,7 @@ def run_ultravox(question: str, audio_count: int):
34 tokenize=False,
35 add_generation_prompt=True)
36
37 - llm = LLM(model=model_name,
38 - enforce_eager=True,
39 - enable_chunked_prefill=False,
40 - max_model_len=8192,
41 - limit_mm_per_prompt={"audio": audio_count})
37 + llm = LLM(model=model_name, limit_mm_per_prompt={"audio": audio_count})
38 stop_token_ids = None
39 return llm, prompt, stop_token_ids
40
tests/kernels/utils.py
+2
@@ -869,6 +869,7 @@ def make_test_metadata(
869 return attn_backend.make_metadata(
870 num_prefills=num_prefills,
871 slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
872 + multi_modal_placeholder_index_maps=None,
873 num_prefill_tokens=num_prefill_tokens,
874 num_decode_tokens=num_decode_tokens,
875 seq_lens=seq_lens,
@@ -914,6 +915,7 @@ def make_test_metadata(
915 return attn_backend.make_metadata(
916 num_prefills=num_prefills,
917 slot_mapping=kv_mmap.slot_mapping,
918 + multi_modal_placeholder_index_maps=None,
919 num_prefill_tokens=num_prefill_tokens,
920 num_decode_tokens=num_decode_tokens,
921 seq_lens=seq_lens,
tests/models/decoder_only/audio_language/test_ultravox.py
+74 -17
@@ -2,8 +2,10 @@ from typing import List, Optional, Tuple, Type
2
3 import numpy as np
4 import pytest
5 +import pytest_asyncio
6 from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8 +from tests.utils import RemoteOpenAIServer
9 from vllm.sequence import SampleLogprobs
10 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
@@ -17,6 +19,13 @@ AudioTuple = Tuple[np.ndarray, int]
19 VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
20 HF_PLACEHOLDER = "<|audio|>"
21
22 +CHUNKED_PREFILL_KWARGS = {
23 + "enable_chunked_prefill": True,
24 + "max_num_seqs": 2,
25 + # Use a very small limit to exercise chunked prefill.
26 + "max_num_batched_tokens": 16
27 +}
28 +
29
30 @pytest.fixture(scope="session")
31 def audio_assets():
@@ -30,6 +39,26 @@ def audio(request):
39 return AudioAsset(request.param)
40
41
42 +@pytest.fixture(params=({}, CHUNKED_PREFILL_KWARGS))
43 +def server(request, audio_assets):
44 + args = [
45 + "--dtype=bfloat16", "--max-model-len=4096", "--enforce-eager",
46 + f"--limit-mm-per-prompt=audio={len(audio_assets)}"
47 + ] + [
48 + f"--{key.replace('_','-')}={value}"
49 + for key, value in request.param.items()
50 + ]
51 +
52 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
53 + yield remote_server
54 +
55 +
56 +@pytest_asyncio.fixture
57 +async def client(server):
58 + async with server.get_async_client() as async_client:
59 + yield async_client
60 +
61 +
62 def _get_prompt(audio_count, question, placeholder):
63 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
64 placeholder = f"{placeholder}\n" * audio_count
@@ -68,8 +97,7 @@ def run_test(
97 dtype: str,
98 max_tokens: int,
99 num_logprobs: int,
71 - tensor_parallel_size: int,
72 - distributed_executor_backend: Optional[str] = None,
100 + **kwargs,
101 ):
102 """Inference result should be the same between hf and vllm."""
103 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
@@ -79,11 +107,8 @@ def run_test(
107 # if we run HF first, the cuda initialization will be done and it
108 # will hurt multiprocessing backend with fork method (the default method).
109
82 - with vllm_runner(model,
83 - dtype=dtype,
84 - tensor_parallel_size=tensor_parallel_size,
85 - distributed_executor_backend=distributed_executor_backend,
86 - enforce_eager=True) as vllm_model:
110 + with vllm_runner(model, dtype=dtype, enforce_eager=True,
111 + **kwargs) as vllm_model:
112 vllm_outputs_per_audio = [
113 vllm_model.generate_greedy_logprobs([vllm_prompt],
114 max_tokens,
@@ -135,18 +160,16 @@ def run_multi_audio_test(
160 dtype: str,
161 max_tokens: int,
162 num_logprobs: int,
138 - tensor_parallel_size: int,
139 - distributed_executor_backend: Optional[str] = None,
163 + **kwargs,
164 ):
165 with vllm_runner(model,
166 dtype=dtype,
143 - tensor_parallel_size=tensor_parallel_size,
144 - distributed_executor_backend=distributed_executor_backend,
167 enforce_eager=True,
168 limit_mm_per_prompt={
169 "audio":
170 max((len(audio) for _, audio in prompts_and_audios))
149 - }) as vllm_model:
171 + },
172 + **kwargs) as vllm_model:
173 vllm_outputs = vllm_model.generate_greedy_logprobs(
174 [prompt for prompt, _ in prompts_and_audios],
175 max_tokens,
@@ -162,8 +185,9 @@ def run_multi_audio_test(
185 @pytest.mark.parametrize("dtype", ["half"])
186 @pytest.mark.parametrize("max_tokens", [128])
187 @pytest.mark.parametrize("num_logprobs", [5])
188 +@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
189 def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
166 - num_logprobs: int) -> None:
190 + num_logprobs: int, vllm_kwargs: dict) -> None:
191
192 vllm_prompt = _get_prompt(1, "Describe the audio above.", VLLM_PLACEHOLDER)
193 hf_prompt = _get_prompt(1, "Describe the audio above.", HF_PLACEHOLDER)
@@ -175,7 +199,7 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
199 dtype=dtype,
200 max_tokens=max_tokens,
201 num_logprobs=num_logprobs,
178 - tensor_parallel_size=1,
202 + **vllm_kwargs,
203 )
204
205
@@ -183,9 +207,10 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
207 @pytest.mark.parametrize("dtype", ["half"])
208 @pytest.mark.parametrize("max_tokens", [128])
209 @pytest.mark.parametrize("num_logprobs", [5])
210 +@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
211 def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
187 - max_tokens: int,
188 - num_logprobs: int) -> None:
212 + max_tokens: int, num_logprobs: int,
213 + vllm_kwargs: dict) -> None:
214
215 vllm_prompt = _get_prompt(len(audio_assets),
216 "Describe each of the audios above.",
@@ -198,5 +223,37 @@ def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
223 dtype=dtype,
224 max_tokens=max_tokens,
225 num_logprobs=num_logprobs,
201 - tensor_parallel_size=1,
226 + **vllm_kwargs,
227 )
228 +
229 +
230 +@pytest.mark.asyncio
231 +async def test_online_inference(client, audio_assets):
232 + """Exercises online inference with/without chunked prefill enabled."""
233 +
234 + messages = [{
235 + "role":
236 + "user",
237 + "content": [
238 + *[{
239 + "type": "audio_url",
240 + "audio_url": {
241 + "url": audio.url
242 + }
243 + } for audio in audio_assets],
244 + {
245 + "type":
246 + "text",
247 + "text":
248 + f"What's happening in these {len(audio_assets)} audio clips?"
249 + },
250 + ],
251 + }]
252 +
253 + chat_completion = await client.chat.completions.create(model=MODEL_NAME,
254 + messages=messages,
255 + max_tokens=10)
256 +
257 + assert len(chat_completion.choices) == 1
258 + choice = chat_completion.choices[0]
259 + assert choice.finish_reason == "length"
tests/multimodal/test_processor_kwargs.py
+7 -7
@@ -5,8 +5,8 @@ from unittest.mock import patch
5 import pytest
6 import torch
7
8 -from vllm.inputs import DecoderOnlyInputs, InputContext, token_inputs
9 -from vllm.inputs.registry import InputRegistry
8 +from vllm.inputs import (DecoderOnlyInputs, DummyData, InputContext,
9 + InputRegistry, token_inputs)
10 from vllm.multimodal import MultiModalRegistry
11 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
12
@@ -56,7 +56,7 @@ def use_dummy_data_mock():
56 num_crops=DEFAULT_NUM_CROPS):
57 seq_data = SequenceData(
58 array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
59 - return seq_data, None
59 + return DummyData(seq_data, None)
60
61 with patch(
62 "vllm.inputs.registry.InputRegistry._default_dummy_data_factory",
@@ -177,9 +177,9 @@ def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
177 # NOTE: seq_len is thrown away here since this will leverage the
178 # default dummy data factory that we have patched in, whose seq
179 # len is solely dependent on the value of the mm_processor_kwargs.
180 - seq_data, _ = dummy_registry.dummy_data_for_profiling(
180 + dummy_data = dummy_registry.dummy_data_for_profiling(
181 ctx.model_config, seq_len=-1, mm_registry=mm_registry)
182 - assert len(seq_data.prompt_token_ids) == expected_seq_count
182 + assert len(dummy_data.seq_data.prompt_token_ids) == expected_seq_count
183
184
185 @pytest.mark.parametrize(
@@ -206,9 +206,9 @@ def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
206 # NOTE: seq_len is thrown away here since this will leverage the
207 # default dummy data factory that we have patched in, whose seq
208 # len is solely dependent on the value of the mm_processor_kwargs.
209 - seq_data, _ = dummy_registry.dummy_data_for_profiling(
209 + dummy_data = dummy_registry.dummy_data_for_profiling(
210 ctx.model_config, seq_len=-1, mm_registry=mm_registry)
211 - assert len(seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
211 + assert len(dummy_data.seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
212
213
214 ### Test overrides for the max token count per multimodal instance
tests/multimodal/test_utils.py
+45 -12
@@ -92,18 +92,50 @@ def test_repeat_and_pad_placeholder_tokens(model):
92 tokenizer = AutoTokenizer.from_pretrained(model)
93
94 test_cases = [
95 - ("<image>", 2, "<image><image>", [32000, 32000]),
96 - ("<image><image>", 2, "<image><image><image>", [32000, 32000, 32000]),
97 - ("<image><image>", [3, 2], "<image><image><image><image><image>",
98 - [32000, 32000, 32000, 32000, 32000]),
99 - ("Image:<image>Image:<image>!", [3, 2],
100 - "Image:<image><image><image>Image:<image><image>!",
101 - [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918]),
102 - ("<image>", [3, 2], "<image><image><image>", [32000, 32000, 32000]),
103 - ]
104 -
105 - for prompt, repeat_count, expected_prompt, expected_token_ids in test_cases:
106 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
95 + (
96 + "<image>",
97 + 2,
98 + "<image><image>",
99 + [32000, 32000],
100 + [{ "offset": 0, "length": 2 }],
101 + ),
102 + (
103 + "<image><image>",
104 + 2,
105 + "<image><image><image>",
106 + [32000, 32000, 32000],
107 + [{ "offset": 0, "length": 2 }]),
108 + (
109 + "<image><image>",
110 + [3, 2],
111 + "<image><image><image><image><image>",
112 + [32000, 32000, 32000, 32000, 32000],
113 + [{ "offset": 0, "length": 3 }, { "offset": 3, "length": 2 }],
114 + ),
115 + (
116 + "Image:<image>Image:<image>!",
117 + [3, 2],
118 + "Image:<image><image><image>Image:<image><image>!",
119 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
120 + [{ "offset": 2, "length": 3 }, { "offset": 7, "length": 2 }],
121 + ),
122 + (
123 + "<image>",
124 + [3, 2],
125 + "<image><image><image>",
126 + [32000, 32000, 32000],
127 + [{ "offset": 0, "length": 3 }],
128 + ),
129 + ] # yapf: disable
130 +
131 + for (
132 + prompt,
133 + repeat_count,
134 + expected_prompt,
135 + expected_token_ids,
136 + expected_ranges,
137 + ) in test_cases:
138 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
139 tokenizer=tokenizer,
140 prompt=prompt,
141 prompt_token_ids=tokenizer.encode(prompt,
@@ -113,3 +145,4 @@ def test_repeat_and_pad_placeholder_tokens(model):
145 )
146 assert new_prompt == expected_prompt
147 assert new_token_ids == expected_token_ids
148 + assert ranges == expected_ranges
tests/worker/test_model_input.py
+3
@@ -73,6 +73,7 @@ def test_model_runner_input():
73 num_prefill_tokens=2,
74 num_decode_tokens=3,
75 slot_mapping=torch.zeros(1),
76 + multi_modal_placeholder_index_maps=None,
77 )
78 model_input = ModelInputForGPUWithSamplingMetadata(
79 input_tokens=torch.ones(10),
@@ -124,6 +125,7 @@ def test_embedding_model_runner_input():
125 num_prefill_tokens=2,
126 num_decode_tokens=3,
127 slot_mapping=torch.zeros(1),
128 + multi_modal_placeholder_index_maps=None,
129 )
130 model_input = ModelInputForGPUWithPoolingMetadata(
131 input_tokens=torch.ones(10),
@@ -174,6 +176,7 @@ def test_multi_step_model_runner_input():
176 num_prefill_tokens=2,
177 num_decode_tokens=3,
178 slot_mapping=torch.zeros(1),
179 + multi_modal_placeholder_index_maps=None,
180 )
181 frozen_model_input = ModelInputForGPUWithSamplingMetadata(
182 input_tokens=torch.ones(10),
vllm/attention/backends/abstract.py
+11
@@ -7,6 +7,8 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
7
8 import torch
9
10 +from vllm.multimodal import MultiModalPlaceholderMap
11 +
12 if TYPE_CHECKING:
13 from vllm.worker.model_runner_base import (ModelRunnerBase,
14 ModelRunnerInputBase,
@@ -108,6 +110,15 @@ class AttentionMetadata:
110 # in block 0, and 1st slot in block 1, respectively.
111 slot_mapping: torch.Tensor
112
113 + # The index maps that relate multi-modal embeddings to the corresponding
114 + # placeholders.
115 + #
116 + # N.B. These aren't really related to attention and don't belong on this
117 + # type -- this is just a temporary solution to make them available to
118 + # `model_executable`.
119 + multi_modal_placeholder_index_maps: Optional[Dict[
120 + str, MultiModalPlaceholderMap.IndexMap]]
121 +
122 @property
123 @abstractmethod
124 def prefill_metadata(self) -> Optional["AttentionMetadata"]:
vllm/attention/backends/blocksparse_attn.py
+3
@@ -215,6 +215,8 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
215 num_prefill_tokens=self.num_prefill_tokens,
216 num_decode_tokens=0,
217 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
218 + multi_modal_placeholder_index_maps=self.
219 + multi_modal_placeholder_index_maps,
220 seq_lens=self.seq_lens[:self.num_prefills],
221 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
222 max_query_len=self.max_query_len,
@@ -243,6 +245,7 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
245 num_prefill_tokens=0,
246 num_decode_tokens=self.num_decode_tokens,
247 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
248 + multi_modal_placeholder_index_maps=None,
249 seq_lens=None,
250 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
251 max_query_len=None,
vllm/attention/backends/flash_attn.py
+20
@@ -1,4 +1,5 @@
1 """Attention layer with FlashAttention."""
2 +from collections import defaultdict
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
5
@@ -14,6 +15,7 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
15 compute_slot_mapping_start_idx,
16 is_block_tables_empty)
17 from vllm.forward_context import get_forward_context
18 +from vllm.multimodal import MultiModalPlaceholderMap
19 from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
20 make_tensor_with_pad)
21
@@ -169,6 +171,8 @@ class FlashAttentionMetadata(AttentionMetadata):
171 num_prefill_tokens=self.num_prefill_tokens,
172 num_decode_tokens=0,
173 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
174 + multi_modal_placeholder_index_maps=self.
175 + multi_modal_placeholder_index_maps,
176 seq_lens=self.seq_lens[:self.num_prefills],
177 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
178 max_query_len=self.max_query_len,
@@ -198,6 +202,7 @@ class FlashAttentionMetadata(AttentionMetadata):
202 num_prefill_tokens=0,
203 num_decode_tokens=self.num_decode_tokens,
204 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
205 + multi_modal_placeholder_index_maps=None,
206 seq_lens=None,
207 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
208 max_decode_query_len=self.max_decode_query_len,
@@ -297,6 +302,9 @@ class FlashAttentionMetadataBuilder(
302 self.context_lens: List[int] = []
303 self.block_tables: List[List[int]] = []
304 self.curr_seq_lens: List[int] = []
305 + self.multimodal_placeholder_maps: Dict[
306 + str,
307 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
308 self.num_prefills = 0
309 self.num_prefill_tokens = 0
310 self.num_decode_tokens = 0
@@ -327,6 +335,12 @@ class FlashAttentionMetadataBuilder(
335 self.context_lens.append(context_len)
336
337 if is_prompt:
338 + mm_maps = inter_data.multi_modal_placeholder_maps
339 + if mm_maps:
340 + for modality, placeholders in mm_maps.items():
341 + self.multimodal_placeholder_maps[modality].extend(
342 + placeholders)
343 +
344 self.num_prefills += 1
345 self.num_prefill_tokens += token_len
346 self.prefill_seq_lens.append(seq_len)
@@ -449,6 +463,11 @@ class FlashAttentionMetadataBuilder(
463 seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
464 dtype=torch.int32,
465 device=device)
466 + placeholder_index_maps = {
467 + modality: placeholder_map.index_map()
468 + for modality, placeholder_map in
469 + self.multimodal_placeholder_maps.items()
470 + }
471 torch.cumsum(seq_lens_tensor,
472 dim=0,
473 dtype=seq_start_loc.dtype,
@@ -464,6 +483,7 @@ class FlashAttentionMetadataBuilder(
483 num_prefill_tokens=self.num_prefill_tokens,
484 num_decode_tokens=num_decode_tokens,
485 seq_lens=seq_lens,
486 + multi_modal_placeholder_index_maps=placeholder_index_maps,
487 seq_lens_tensor=seq_lens_tensor,
488 max_query_len=max_query_len,
489 max_decode_query_len=max_decode_query_len,
vllm/attention/backends/flashinfer.py
+18
@@ -1,7 +1,10 @@
1 +from collections import defaultdict
2 from contextlib import contextmanager
3 from dataclasses import dataclass
4 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
5
6 +from vllm.multimodal import MultiModalPlaceholderMap
7 +
8 try:
9 from flashinfer import BatchDecodeWithPagedKVCacheWrapper
10 from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
@@ -215,6 +218,7 @@ class FlashInferState(AttentionState):
218 attn_metadata = self.runner.attn_backend.make_metadata(
219 num_prefills=0,
220 slot_mapping=self._graph_slot_mapping[:batch_size],
221 + multi_modal_placeholder_index_maps=None,
222 num_prefill_tokens=0,
223 num_decode_tokens=batch_size,
224 max_prefill_seq_len=0,
@@ -470,6 +474,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
474 self.context_lens: List[int] = []
475 self.block_tables: List[List[int]] = []
476 self.curr_seq_lens: List[int] = []
477 + self.multimodal_placeholder_maps: Dict[
478 + str,
479 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
480 self.num_prefills = 0
481 self.num_prefill_tokens = 0
482 self.num_decode_tokens = 0
@@ -519,6 +526,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
526 inter_data.curr_sliding_window_blocks):
527 self.context_lens.append(context_len)
528 if is_prompt:
529 + mm_maps = inter_data.multi_modal_placeholder_maps
530 + if mm_maps:
531 + for modality, placeholders in mm_maps.items():
532 + self.multimodal_placeholder_maps[modality].extend(
533 + placeholders)
534 self.num_prefills += 1
535 self.num_prefill_tokens += token_len
536 self.prefill_seq_lens.append(seq_len)
@@ -651,6 +663,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
663 seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
664 dtype=torch.int32,
665 device=device)
666 + placeholder_index_maps = {
667 + modality: placeholder_map.index_map()
668 + for modality, placeholder_map in
669 + self.multimodal_placeholder_maps.items()
670 + }
671 torch.cumsum(seq_lens_tensor,
672 dim=0,
673 dtype=seq_start_loc.dtype,
@@ -694,6 +711,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
711 decode_query_len=decode_query_len,
712 num_prefills=self.num_prefills,
713 slot_mapping=slot_mapping_tensor,
714 + multi_modal_placeholder_index_maps=placeholder_index_maps,
715 num_prefill_tokens=self.num_prefill_tokens,
716 num_decode_tokens=num_decode_tokens,
717 max_prefill_seq_len=max_prefill_seq_len,
vllm/attention/backends/placeholder_attn.py
+21 -1
@@ -1,5 +1,6 @@
1 +from collections import defaultdict
2 from dataclasses import dataclass
2 -from typing import TYPE_CHECKING, List, Optional, Tuple, Type
3 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type
4
5 import torch
6
@@ -7,6 +8,7 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 AttentionMetadata,
9 AttentionMetadataBuilder)
10 from vllm.attention.backends.utils import CommonAttentionState
11 +from vllm.multimodal import MultiModalPlaceholderMap
12
13 if TYPE_CHECKING:
14 from vllm.worker.model_runner import ModelInputForGPUBuilder
@@ -135,6 +137,8 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
137 num_prefill_tokens=self.num_prefill_tokens,
138 num_decode_tokens=0,
139 slot_mapping=slot_mapping,
140 + multi_modal_placeholder_index_maps=self.
141 + multi_modal_placeholder_index_maps,
142 seq_lens=self.seq_lens[:self.num_prefills],
143 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
144 max_decode_query_len=0,
@@ -167,6 +171,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
171 num_prefill_tokens=0,
172 num_decode_tokens=self.num_decode_tokens,
173 slot_mapping=slot_mapping,
174 + multi_modal_placeholder_index_maps=None,
175 seq_lens=None,
176 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
177 max_decode_query_len=self.max_decode_query_len,
@@ -189,6 +194,9 @@ class PlaceholderAttentionMetadataBuilder(
194 self.prefill_seq_lens: List[int] = []
195 self.context_lens: List[int] = []
196 self.curr_seq_lens: List[int] = []
197 + self.multimodal_placeholder_maps: Dict[
198 + str,
199 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
200 self.num_prefills = 0
201 self.num_prefill_tokens = 0
202 self.num_decode_tokens = 0
@@ -213,6 +221,12 @@ class PlaceholderAttentionMetadataBuilder(
221 self.context_lens.append(context_len)
222
223 if is_prompt:
224 + mm_maps = inter_data.multi_modal_placeholder_maps
225 + if mm_maps:
226 + for modality, placeholders in mm_maps.items():
227 + self.multimodal_placeholder_maps[modality].extend(
228 + placeholders)
229 +
230 self.num_prefills += 1
231 self.num_prefill_tokens += token_len
232 self.prefill_seq_lens.append(seq_len)
@@ -280,6 +294,11 @@ class PlaceholderAttentionMetadataBuilder(
294 seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
295 dtype=torch.int32,
296 device=device)
297 + placeholder_index_maps = {
298 + modality: placeholder_map.index_map()
299 + for modality, placeholder_map in
300 + self.multimodal_placeholder_maps.items()
301 + }
302 torch.cumsum(seq_lens_tensor,
303 dim=0,
304 dtype=seq_start_loc.dtype,
@@ -296,6 +315,7 @@ class PlaceholderAttentionMetadataBuilder(
315 return PlaceholderAttentionMetadata(
316 num_prefills=self.num_prefills,
317 slot_mapping=slot_mapping,
318 + multi_modal_placeholder_index_maps=placeholder_index_maps,
319 num_prefill_tokens=self.num_prefill_tokens,
320 num_decode_tokens=num_decode_tokens,
321 seq_lens=seq_lens,
vllm/attention/backends/rocm_flash_attn.py
+3
@@ -150,6 +150,8 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
150 num_prefill_tokens=self.num_prefill_tokens,
151 num_decode_tokens=0,
152 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
153 + multi_modal_placeholder_index_maps=self.
154 + multi_modal_placeholder_index_maps,
155 seq_lens=self.seq_lens[:self.num_prefills],
156 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
157 max_query_len=self.max_query_len,
@@ -178,6 +180,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
180 num_prefill_tokens=0,
181 num_decode_tokens=self.num_decode_tokens,
182 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
183 + multi_modal_placeholder_index_maps=None,
184 seq_lens=None,
185 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
186 max_query_len=None,
vllm/attention/backends/utils.py
+18
@@ -1,4 +1,5 @@
1 """Attention backend utils"""
2 +from collections import defaultdict
3 from contextlib import contextmanager
4 from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
5
@@ -7,6 +8,7 @@ import torch
8
9 from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
10 AttentionState)
11 +from vllm.multimodal import MultiModalPlaceholderMap
12 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
13
14 if TYPE_CHECKING:
@@ -123,6 +125,9 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
125 self.context_lens: List[int] = []
126 self.block_tables: List[List[int]] = []
127 self.curr_seq_lens: List[int] = []
128 + self.multimodal_placeholder_maps: Dict[
129 + str,
130 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
131 self.num_prefills = 0
132 self.num_prefill_tokens = 0
133 self.num_decode_tokens = 0
@@ -147,6 +152,12 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
152 inter_data.curr_sliding_window_blocks):
153 self.context_lens.append(context_len)
154 if is_prompt:
155 + mm_maps = inter_data.multi_modal_placeholder_maps
156 + if mm_maps:
157 + for modality, placeholders in mm_maps.items():
158 + self.multimodal_placeholder_maps[modality].extend(
159 + placeholders)
160 +
161 self.num_prefills += 1
162 self.num_prefill_tokens += token_len
163 self.prefill_seq_lens.append(seq_len)
@@ -242,6 +253,11 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
253 seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
254 dtype=torch.int32,
255 device=device)
256 + placeholder_index_maps = {
257 + modality: placeholder_map.index_map()
258 + for modality, placeholder_map in
259 + self.multimodal_placeholder_maps.items()
260 + }
261 torch.cumsum(seq_lens_tensor,
262 dim=0,
263 dtype=seq_start_loc.dtype,
@@ -254,6 +270,7 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
270 return self._metadata_cls( # type: ignore
271 num_prefills=self.num_prefills,
272 slot_mapping=slot_mapping_tensor,
273 + multi_modal_placeholder_index_maps=placeholder_index_maps,
274 num_prefill_tokens=self.num_prefill_tokens,
275 num_decode_tokens=num_decode_tokens,
276 seq_lens=seq_lens,
@@ -305,6 +322,7 @@ class CommonAttentionState(AttentionState):
322 num_prefill_tokens=0,
323 num_decode_tokens=batch_size,
324 slot_mapping=self._graph_slot_mapping[:batch_size],
325 + multi_modal_placeholder_index_maps=None,
326 seq_lens=None,
327 seq_lens_tensor=self._graph_seq_lens[:batch_size],
328 max_query_len=1,
vllm/attention/backends/xformers.py
+3
@@ -212,6 +212,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
212 num_prefill_tokens=self.num_prefill_tokens,
213 num_decode_tokens=0,
214 slot_mapping=slot_mapping,
215 + multi_modal_placeholder_index_maps=self.
216 + multi_modal_placeholder_index_maps,
217 seq_lens=seq_lens,
218 seq_lens_tensor=seq_lens_tensor,
219 max_query_len=self.max_query_len,
@@ -255,6 +257,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
257 num_prefill_tokens=0,
258 num_decode_tokens=self.num_decode_tokens,
259 slot_mapping=slot_mapping,
260 + multi_modal_placeholder_index_maps=None,
261 seq_lens_tensor=seq_lens_tensor,
262 max_prefill_seq_len=0,
263 max_decode_seq_len=self.max_decode_seq_len,
vllm/core/scheduler.py
+2
@@ -1308,6 +1308,8 @@ class Scheduler:
1308 # `multi_modal_data` will be None.
1309 multi_modal_data=seq_group.multi_modal_data
1310 if scheduler_outputs.num_prefill_groups > 0 else None,
1311 + multi_modal_placeholders=seq_group.multi_modal_placeholders
1312 + if scheduler_outputs.num_prefill_groups > 0 else None,
1313 mm_processor_kwargs=seq_group.mm_processor_kwargs,
1314 prompt_adapter_request=seq_group.prompt_adapter_request,
1315 )
vllm/inputs/__init__.py
+2 -1
@@ -3,7 +3,7 @@ from .data import (DecoderOnlyInputs, EncoderDecoderInputs,
3 SingletonPrompt, TextPrompt, TokenInputs, TokensPrompt,
4 build_explicit_enc_dec_prompt, to_enc_dec_tuple_list,
5 token_inputs, zip_enc_dec_prompts)
6 -from .registry import InputContext, InputRegistry
6 +from .registry import DummyData, InputContext, InputRegistry
7
8 INPUT_REGISTRY = InputRegistry()
9 """
@@ -29,6 +29,7 @@ __all__ = [
29 "to_enc_dec_tuple_list",
30 "zip_enc_dec_prompts",
31 "INPUT_REGISTRY",
32 + "DummyData",
33 "InputContext",
34 "InputRegistry",
35 ]
vllm/inputs/data.py
+10 -1
@@ -4,7 +4,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
4 from typing_extensions import NotRequired, TypedDict, TypeVar
5
6 if TYPE_CHECKING:
7 - from vllm.multimodal import MultiModalDataDict
7 + from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
8
9
10 class TextPrompt(TypedDict):
@@ -136,6 +136,12 @@ class TokenInputs(TypedDict):
136 if the model supports it.
137 """
138
139 + multi_modal_placeholders: NotRequired[
140 + Optional["MultiModalPlaceholderDict"]]
141 + """
142 + Placeholder ranges for the multi-modal data.
143 + """
144 +
145 mm_processor_kwargs: NotRequired[Optional[Dict[str, Any]]]
146 """
147 Optional multi-modal processor kwargs to be forwarded to the
@@ -149,6 +155,7 @@ def token_inputs(
155 prompt_token_ids: List[int],
156 prompt: Optional[str] = None,
157 multi_modal_data: Optional["MultiModalDataDict"] = None,
158 + multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None,
159 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
160 ) -> TokenInputs:
161 """Construct :class:`TokenInputs` from optional values."""
@@ -158,6 +165,8 @@ def token_inputs(
165 inputs["prompt"] = prompt
166 if multi_modal_data is not None:
167 inputs["multi_modal_data"] = multi_modal_data
168 + if multi_modal_placeholders is not None:
169 + inputs["multi_modal_placeholders"] = multi_modal_placeholders
170 if mm_processor_kwargs is not None:
171 inputs["mm_processor_kwargs"] = mm_processor_kwargs
172
vllm/inputs/registry.py
+23 -17
@@ -1,8 +1,8 @@
1 import functools
2 from collections import UserDict
3 from dataclasses import dataclass
4 -from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
5 - Protocol, Tuple, Type)
4 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5 + Optional, Protocol, Type)
6
7 from torch import nn
8 from transformers import PretrainedConfig
@@ -16,7 +16,8 @@ from .data import DecoderOnlyInputs
16
17 if TYPE_CHECKING:
18 from vllm.config import ModelConfig
19 - from vllm.multimodal import MultiModalDataDict, MultiModalRegistry
19 + from vllm.multimodal import (MultiModalDataDict, MultiModalPlaceholderDict,
20 + MultiModalRegistry)
21 from vllm.sequence import SequenceData
22
23 logger = init_logger(__name__)
@@ -63,6 +64,14 @@ class InputContext:
64 N = TypeVar("N", bound=Type[nn.Module])
65
66
67 +class DummyData(NamedTuple):
68 + """Dummy data used for profiling."""
69 +
70 + seq_data: "SequenceData"
71 + multi_modal_data: Optional["MultiModalDataDict"] = None
72 + multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None
73 +
74 +
75 class DummyDataFactory(Protocol):
76
77 def __call__(
@@ -71,7 +80,7 @@ class DummyDataFactory(Protocol):
80 seq_len: int,
81 mm_counts: Mapping[str, int],
82 **mm_processor_kwargs: Any,
74 - ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
83 + ) -> DummyData:
84 """
85 Create dummy data to be inputted into the model.
86
@@ -123,7 +132,7 @@ class InputRegistry:
132 ctx: InputContext,
133 seq_len: int,
134 mm_counts: Mapping[str, int],
126 - ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
135 + ) -> DummyData:
136 """
137 The default dummy data factory represents the longest possible text
138 that can be inputted to the model.
@@ -134,10 +143,7 @@ class InputRegistry:
143 # Avoid circular import
144 from vllm.sequence import SequenceData
145
137 - dummy_seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
138 - dummy_multi_modal_data = None
139 -
140 - return dummy_seq_data, dummy_multi_modal_data
146 + return DummyData(SequenceData.from_prompt_token_counts((0, seq_len)))
147
148 def register_dummy_data(self, factory: DummyDataFactory):
149 """
@@ -195,7 +201,7 @@ class InputRegistry:
201 seq_len: int,
202 mm_registry: "MultiModalRegistry",
203 is_encoder_data: bool = False,
198 - ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
204 + ) -> DummyData:
205 """
206 Create dummy data for profiling the memory usage of a model.
207
@@ -220,12 +226,12 @@ class InputRegistry:
226 mm_processor_kwargs = get_allowed_kwarg_only_overrides(
227 dummy_factory, overrides=model_config.mm_processor_kwargs)
228
223 - seq_data, mm_data = dummy_factory(InputContext(model_config), seq_len,
224 - _MultiModalCounts(mm_counts),
225 - **mm_processor_kwargs)
229 + dummy_data = dummy_factory(InputContext(model_config), seq_len,
230 + _MultiModalCounts(mm_counts),
231 + **mm_processor_kwargs)
232
233 # Having more tokens is over-conservative but otherwise fine
228 - num_tokens = seq_data.prompt_token_ids
234 + num_tokens = dummy_data.seq_data.prompt_token_ids
235 if len(num_tokens) < seq_len:
236 if is_encoder_data:
237 print_warning_once(
@@ -235,15 +241,15 @@ class InputRegistry:
241 raise AssertionError(
242 f"Expected at least {seq_len} dummy tokens for profiling, "
243 f"but found {len(num_tokens)} tokens instead.")
238 - if mm_data is not None:
239 - for k, v in mm_data.items():
244 + if dummy_data.multi_modal_data is not None:
245 + for k, v in dummy_data.multi_modal_data.items():
246 num_items = len(v) if isinstance(v, list) else 1
247 num_expected = mm_counts[k]
248 assert num_items >= num_expected, (
249 f"Expected at least {num_expected} dummy '{k}' instances "
250 f"for profiling, but found {num_items} instances instead.")
251
246 - return seq_data, mm_data
252 + return dummy_data
253
254 def _default_input_processor(
255 self,
vllm/model_executor/models/blip.py
+8 -2
@@ -98,6 +98,11 @@ def input_processor_for_blip(
98 if multi_modal_data is None or "image" not in multi_modal_data:
99 return inputs
100
101 + if "multi_modal_placeholders" in inputs and "image" in inputs[
102 + "multi_modal_placeholders"]:
103 + # The inputs already have placeholders.
104 + return inputs
105 +
106 tokenizer = cached_get_tokenizer(model_config.tokenizer)
107
108 if image_feature_size_override is None:
@@ -105,7 +110,7 @@ def input_processor_for_blip(
110 else:
111 image_feature_size = image_feature_size_override
112
108 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
113 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
114 tokenizer,
115 inputs.get("prompt"),
116 inputs["prompt_token_ids"],
@@ -116,7 +121,8 @@ def input_processor_for_blip(
121 # NOTE: Create a defensive copy of the original inputs
122 return token_inputs(prompt_token_ids=new_token_ids,
123 prompt=new_prompt,
119 - multi_modal_data=multi_modal_data)
124 + multi_modal_data=multi_modal_data,
125 + multi_modal_placeholders={"image": ranges})
126
127
128 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
vllm/model_executor/models/blip2.py
+10 -5
@@ -9,13 +9,14 @@ from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, MultiModalConfig
12 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
13 - token_inputs)
12 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13 + InputContext, token_inputs)
14 from vllm.model_executor.layers.activation import get_act_fn
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.utils import consecutive_placeholder_ranges
20 from vllm.sequence import IntermediateTensors, SequenceData
21
22 from .blip import (BlipVisionModel, dummy_image_for_blip,
@@ -425,7 +426,11 @@ def dummy_seq_data_for_blip2(
426 return SequenceData.from_prompt_token_counts(
427 (image_token_id, image_feature_size * num_images),
428 (0, seq_len - image_feature_size * num_images),
428 - )
429 + ), {
430 + "image":
431 + consecutive_placeholder_ranges(num_items=num_images,
432 + item_size=image_feature_size)
433 + }
434
435
436 def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
@@ -434,7 +439,7 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
439 vision_config = hf_config.vision_config
440 num_images = mm_counts["image"]
441
437 - seq_data = dummy_seq_data_for_blip2(
442 + seq_data, ranges = dummy_seq_data_for_blip2(
443 hf_config,
444 seq_len,
445 num_images,
@@ -444,7 +449,7 @@ def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
449 if isinstance(vision_config, Blip2VisionConfig):
450 mm_data = dummy_image_for_blip(vision_config, num_images)
451
447 - return seq_data, mm_data
452 + return DummyData(seq_data, mm_data, ranges)
453
454 msg = f"Unsupported vision config: {type(vision_config)}"
455 raise NotImplementedError(msg)
vllm/model_executor/models/chameleon.py
+16 -6
@@ -11,8 +11,8 @@ from transformers import ChameleonConfig, ChameleonVQVAEConfig
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
13 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15 - token_inputs)
14 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 + InputContext, token_inputs)
16 from vllm.model_executor.layers.activation import SiluAndMul
17 from vllm.model_executor.layers.layernorm import RMSNorm
18 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -30,6 +30,7 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
30 from vllm.model_executor.utils import set_weight_attrs
31 from vllm.multimodal import MULTIMODAL_REGISTRY
32 from vllm.multimodal.utils import (cached_get_tokenizer,
33 + consecutive_placeholder_ranges,
34 repeat_and_pad_placeholder_tokens)
35 from vllm.sequence import IntermediateTensors, SequenceData
36 from vllm.utils import print_warning_once
@@ -73,7 +74,11 @@ def dummy_seq_data_for_chameleon(
74 return SequenceData.from_prompt_token_counts(
75 (image_token_id, image_feature_size * num_images),
76 (0, seq_len - image_feature_size * num_images),
76 - )
77 + ), {
78 + "image":
79 + consecutive_placeholder_ranges(num_items=num_images,
80 + item_size=image_feature_size)
81 + }
82
83
84 def dummy_image_for_chameleon(
@@ -97,14 +102,14 @@ def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
102 mm_counts: Mapping[str, int]):
103 num_images = mm_counts["image"]
104
100 - seq_data = dummy_seq_data_for_chameleon(
105 + seq_data, ranges = dummy_seq_data_for_chameleon(
106 seq_len,
107 num_images,
108 image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
109 )
110
111 mm_data = dummy_image_for_chameleon(num_images)
107 - return seq_data, mm_data
112 + return DummyData(seq_data, mm_data, ranges)
113
114
115 def input_processor_for_chameleon(ctx: InputContext,
@@ -120,9 +125,14 @@ def input_processor_for_chameleon(ctx: InputContext,
125 if multi_modal_data is None or "image" not in multi_modal_data:
126 return inputs
127
128 + if "multi_modal_placeholders" in inputs and "image" in inputs[
129 + "multi_modal_placeholders"]:
130 + # The inputs already have placeholders.
131 + return inputs
132 +
133 model_config = ctx.model_config
134 tokenizer = cached_get_tokenizer(model_config.tokenizer)
125 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
135 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
136 tokenizer,
137 inputs.get("prompt"),
138 inputs["prompt_token_ids"],
vllm/model_executor/models/clip.py
+21 -11
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.multimodal.utils import (cached_get_tokenizer,
22 + consecutive_placeholder_ranges,
23 repeat_and_pad_placeholder_tokens)
24 from vllm.sequence import SequenceData
25
@@ -49,14 +50,13 @@ def get_max_clip_image_tokens(hf_config: CLIPVisionConfig) -> int:
50 return get_clip_image_feature_size(hf_config)
51
52
52 -def dummy_seq_data_for_clip(
53 - hf_config: CLIPVisionConfig,
54 - seq_len: int,
55 - num_images: int,
56 - *,
57 - image_token_id: int,
58 - image_feature_size_override: Optional[int] = None,
59 -):
53 +def dummy_seq_data_for_clip(hf_config: CLIPVisionConfig,
54 + seq_len: int,
55 + num_images: int,
56 + *,
57 + image_token_id: int,
58 + image_feature_size_override: Optional[int] = None,
59 + mm_key: str = "image"):
60 if image_feature_size_override is None:
61 image_feature_size = get_clip_image_feature_size(hf_config)
62 else:
@@ -65,7 +65,11 @@ def dummy_seq_data_for_clip(
65 return SequenceData.from_prompt_token_counts(
66 (image_token_id, image_feature_size * num_images),
67 (0, seq_len - image_feature_size * num_images),
68 - )
68 + ), {
69 + mm_key:
70 + consecutive_placeholder_ranges(num_items=num_images,
71 + item_size=image_feature_size)
72 + }
73
74
75 def dummy_image_for_clip(
@@ -117,6 +121,11 @@ def input_processor_for_clip(
121 if multi_modal_data is None or "image" not in multi_modal_data:
122 return inputs
123
124 + if "multi_modal_placeholders" in inputs and "image" in inputs[
125 + "multi_modal_placeholders"]:
126 + # The inputs already have placeholders.
127 + return inputs
128 +
129 tokenizer = cached_get_tokenizer(model_config.tokenizer)
130
131 if image_feature_size_override is None:
@@ -130,7 +139,7 @@ def input_processor_for_clip(
139 else:
140 image_feature_size = image_feature_size_override
141
133 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
142 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
143 tokenizer,
144 inputs.get("prompt"),
145 inputs["prompt_token_ids"],
@@ -141,7 +150,8 @@ def input_processor_for_clip(
150 # NOTE: Create a defensive copy of the original inputs
151 return token_inputs(prompt_token_ids=new_token_ids,
152 prompt=new_prompt,
144 - multi_modal_data=multi_modal_data)
153 + multi_modal_data=multi_modal_data,
154 + multi_modal_placeholders={"image": ranges})
155
156
157 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/fuyu.py
+20 -11
@@ -27,8 +27,8 @@ from transformers import FuyuConfig, FuyuImageProcessor
27
28 from vllm.attention import AttentionMetadata
29 from vllm.config import CacheConfig, MultiModalConfig
30 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
31 - token_inputs)
30 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31 + InputContext, token_inputs)
32 from vllm.model_executor.layers.linear import ColumnParallelLinear
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -37,9 +37,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
38 from vllm.multimodal.base import MultiModalInputs
39 from vllm.multimodal.image import cached_get_image_processor
40 -from vllm.multimodal.utils import cached_get_tokenizer
40 +from vllm.multimodal.utils import (cached_get_tokenizer,
41 + consecutive_placeholder_ranges)
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
43 SequenceData)
44 +from vllm.utils import is_list_of
45
46 from .interfaces import SupportsMultiModal, SupportsPP
47 from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
@@ -103,7 +105,11 @@ def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
105 token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106 token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107 [0]) * (seq_len - image_feature_size * num_images)
106 - return SequenceData(token_ids)
108 + return SequenceData(token_ids), {
109 + "image":
110 + consecutive_placeholder_ranges(num_items=num_images,
111 + item_size=image_feature_size)
112 + }
113
114
115 def dummy_image_for_fuyu(
@@ -119,15 +125,15 @@ def dummy_image_for_fuyu(
125 def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
126 mm_counts: Mapping[str, int]):
127 num_images = mm_counts["image"]
122 - seq_data = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
128 + seq_data, ranges = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
129 mm_data = dummy_image_for_fuyu(num_images,
130 image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
131 image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
126 - return seq_data, mm_data
132 + return DummyData(seq_data, mm_data, ranges)
133
134
135 def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
130 - data: Image.Image):
136 + data: List[Image.Image]):
137 image_encoding = image_processor.preprocess(data, return_tensors="pt")
138 batch_images = torch.stack([img[0] for img in image_encoding["images"]
139 ]).unsqueeze(1)
@@ -158,8 +164,10 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
164 model_config = ctx.model_config
165 image_data = multi_modal_data["image"]
166 new_multi_modal_data = {}
167 + image_list = image_data if isinstance(image_data, list) else [image_data]
168 +
169 # process image data
162 - if isinstance(image_data, Image.Image):
170 + if is_list_of(image_list, Image.Image):
171 # Fuyu's image_processor can also finish token padding
172 image_processor: FuyuImageProcessor = cached_get_image_processor(
173 model_config.model)
@@ -171,7 +179,7 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
179 ])
180 new_multi_modal_data["image"] = image_patches
181
174 - elif isinstance(image_data, torch.Tensor):
182 + elif is_list_of(image_list, torch.Tensor):
183 raise NotImplementedError("Embeddings input is not supported yet")
184 else:
185 raise TypeError(f"Invalid image type: {type(image_data)}")
@@ -198,12 +206,13 @@ def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
206
207 def input_mapper_for_fuyu(ctx: InputContext, data: object):
208 model_config = ctx.model_config
201 - if isinstance(data, Image.Image):
209 + data_list = data if isinstance(data, list) else [data]
210 + if is_list_of(data_list, Image.Image):
211 # Fuyu's image_processor can also finish token padding
212 image_processor: FuyuImageProcessor = cached_get_image_processor(
213 model_config.model)
214
206 - model_image_input = _fuyu_image_preprocess(image_processor, data)
215 + model_image_input = _fuyu_image_preprocess(image_processor, data_list)
216 data = torch.stack([
217 image_patch[0]
218 for image_patch in model_image_input["image_patches"]
vllm/model_executor/models/internvl.py
+4 -4
@@ -17,8 +17,8 @@ from transformers import PretrainedConfig
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, MultiModalConfig
20 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21 - token_inputs)
20 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 + InputContext, token_inputs)
22 from vllm.model_executor.layers.quantization import (AWQConfig,
23 QuantizationConfig)
24 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -379,7 +379,7 @@ class InternVLInputPipeline:
379 model_config.tokenizer,
380 trust_remote_code=model_config.trust_remote_code)
381
382 - seq_data = dummy_seq_data_for_clip(
382 + seq_data, ranges = dummy_seq_data_for_clip(
383 hf_config.vision_config,
384 seq_len,
385 num_images,
@@ -398,7 +398,7 @@ class InternVLInputPipeline:
398 image_height_override=max_image_height,
399 )
400
401 - return seq_data, mm_data
401 + return DummyData(seq_data, mm_data, ranges)
402
403
404 input_pipeline = InternVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
vllm/model_executor/models/llava.py
+8 -7
@@ -10,7 +10,8 @@ from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
10
11 from vllm.attention import AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
13 -from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
13 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
14 + InputContext)
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.quantization import QuantizationConfig
17 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -111,7 +112,7 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
112 image_feature_size = get_max_llava_image_tokens(ctx)
113
114 if isinstance(vision_config, CLIPVisionConfig):
114 - seq_data = dummy_seq_data_for_clip(
115 + seq_data, ranges = dummy_seq_data_for_clip(
116 vision_config,
117 seq_len,
118 num_images,
@@ -120,9 +121,9 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
121 )
122
123 mm_data = dummy_image_for_clip(vision_config, num_images)
123 - return seq_data, mm_data
124 + return DummyData(seq_data, mm_data, ranges)
125 elif isinstance(vision_config, SiglipVisionConfig):
125 - seq_data = dummy_seq_data_for_siglip(
126 + seq_data, ranges = dummy_seq_data_for_siglip(
127 vision_config,
128 seq_len,
129 num_images,
@@ -131,9 +132,9 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
132 )
133
134 mm_data = dummy_image_for_siglip(vision_config, num_images)
134 - return seq_data, mm_data
135 + return DummyData(seq_data, mm_data, ranges)
136 elif isinstance(vision_config, PixtralVisionConfig):
136 - seq_data = dummy_seq_data_for_pixtral_hf(
137 + seq_data, ranges = dummy_seq_data_for_pixtral_hf(
138 vision_config,
139 seq_len,
140 num_images,
@@ -142,7 +143,7 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int,
143 )
144
145 mm_data = dummy_image_for_pixtral_hf(vision_config, num_images)
145 - return seq_data, mm_data
146 + return DummyData(seq_data, mm_data, ranges)
147
148 msg = f"Unsupported vision config: {type(vision_config)}"
149 raise NotImplementedError(msg)
vllm/model_executor/models/llava_next.py
+6 -5
@@ -12,7 +12,8 @@ from typing_extensions import NotRequired
12
13 from vllm.attention import AttentionMetadata
14 from vllm.config import CacheConfig, MultiModalConfig, PoolerConfig
15 -from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
15 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
16 + InputContext)
17 from vllm.model_executor.layers.pooler import Pooler, PoolingType
18 from vllm.model_executor.layers.quantization import QuantizationConfig
19 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -180,7 +181,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
181 max_feat_height, max_feat_width = pinpoint
182
183 if isinstance(vision_config, CLIPVisionConfig):
183 - seq_data = dummy_seq_data_for_clip(
184 + seq_data, ranges = dummy_seq_data_for_clip(
185 vision_config,
186 seq_len,
187 num_images,
@@ -195,9 +196,9 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
196 image_height_override=max_feat_height,
197 )
198
198 - return seq_data, mm_data
199 + return DummyData(seq_data, mm_data, ranges)
200 elif isinstance(vision_config, SiglipVisionConfig):
200 - seq_data = dummy_seq_data_for_siglip(
201 + seq_data, ranges = dummy_seq_data_for_siglip(
202 vision_config,
203 seq_len,
204 num_images,
@@ -212,7 +213,7 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int,
213 image_height_override=max_feat_height,
214 )
215
215 - return seq_data, mm_data
216 + return DummyData(seq_data, mm_data, ranges)
217
218 msg = f"Unsupported vision config: {type(vision_config)}"
219 raise NotImplementedError(msg)
vllm/model_executor/models/llava_next_video.py
+17 -8
@@ -11,8 +11,8 @@ from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
11
12 from vllm.attention import AttentionMetadata
13 from vllm.config import CacheConfig, MultiModalConfig
14 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
15 - token_inputs)
14 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 + InputContext, token_inputs)
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -108,33 +108,35 @@ def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
108 video_feature_size = frames_per_video * tokens_per_frame
109
110 if isinstance(vision_config, CLIPVisionConfig):
111 - seq_data = dummy_seq_data_for_clip(
111 + seq_data, ranges = dummy_seq_data_for_clip(
112 vision_config,
113 seq_len,
114 num_videos,
115 image_token_id=hf_config.video_token_index,
116 image_feature_size_override=video_feature_size,
117 + mm_key="video",
118 )
119
120 pil_frame = dummy_image_for_clip(vision_config, num_images=1)
121 np_frame = np.array(pil_frame["image"])
122 mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
123 mm_data = {"video": mm_data_per_video}
123 - return seq_data, mm_data
124 + return DummyData(seq_data, mm_data, ranges)
125 elif isinstance(vision_config, SiglipVisionConfig):
125 - seq_data = dummy_seq_data_for_siglip(
126 + seq_data, ranges = dummy_seq_data_for_siglip(
127 vision_config,
128 seq_len,
129 num_videos,
130 image_token_id=hf_config.video_token_index,
131 image_feature_size_override=video_feature_size,
132 + mm_key="video",
133 )
134
135 pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
136 np_frame = np.array(pil_frame["image"])
137 mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
138 mm_data = {"video": mm_data_per_video}
137 - return seq_data, mm_data
139 + return DummyData(seq_data, mm_data, ranges)
140
141 msg = f"Unsupported vision config: {type(vision_config)}"
142 raise NotImplementedError(msg)
@@ -145,6 +147,12 @@ def input_processor_for_llava_next_video(ctx: InputContext,
147 multi_modal_data = inputs.get("multi_modal_data")
148 if multi_modal_data is None or "video" not in multi_modal_data:
149 return inputs
150 +
151 + if "multi_modal_placeholders" in inputs and "video" in inputs[
152 + "multi_modal_placeholders"]:
153 + # The inputs already have placeholders.
154 + return inputs
155 +
156 video_data = multi_modal_data["video"]
157
158 model_config = ctx.model_config
@@ -160,7 +168,7 @@ def input_processor_for_llava_next_video(ctx: InputContext,
168
169 tokenizer = cached_get_tokenizer(model_config.tokenizer)
170
163 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
171 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
172 tokenizer,
173 inputs.get("prompt"),
174 inputs["prompt_token_ids"],
@@ -170,7 +178,8 @@ def input_processor_for_llava_next_video(ctx: InputContext,
178
179 return token_inputs(prompt_token_ids=new_token_ids,
180 prompt=new_prompt,
173 - multi_modal_data=multi_modal_data)
181 + multi_modal_data=multi_modal_data,
182 + multi_modal_placeholders={"video": ranges})
183
184 elif is_list_of(video_data, np.ndarray):
185 raise NotImplementedError(
vllm/model_executor/models/llava_onevision.py
+11 -10
@@ -15,8 +15,8 @@ from typing_extensions import NotRequired
15
16 from vllm.attention import AttentionMetadata
17 from vllm.config import CacheConfig, MultiModalConfig
18 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
19 - token_inputs)
18 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19 + InputContext, token_inputs)
20 from vllm.model_executor.layers.activation import get_act_fn
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
@@ -218,31 +218,31 @@ def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
218 video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
219
220 if isinstance(vision_config, CLIPVisionConfig):
221 - seq_data = dummy_seq_data_for_clip(
221 + seq_data, ranges = dummy_seq_data_for_clip(
222 vision_config,
223 seq_len,
224 num_videos,
225 image_token_id=hf_config.video_token_index,
226 image_feature_size_override=video_feature_size,
227 - )
227 + mm_key="video")
228
229 mm_data = dummy_video_for_clip(vision_config,
230 num_frames=num_frames,
231 num_videos=num_videos)
232 - return seq_data, mm_data
232 + return DummyData(seq_data, mm_data, ranges)
233 elif isinstance(vision_config, SiglipVisionConfig):
234 - seq_data = dummy_seq_data_for_siglip(
234 + seq_data, ranges = dummy_seq_data_for_siglip(
235 vision_config,
236 seq_len,
237 num_videos,
238 image_token_id=hf_config.video_token_index,
239 image_feature_size_override=video_feature_size,
240 - )
240 + mm_key="video")
241
242 mm_data = dummy_video_for_siglip(vision_config,
243 num_frames=num_frames,
244 num_videos=num_videos)
245 - return seq_data, mm_data
245 + return DummyData(seq_data, mm_data, ranges)
246
247 msg = f"Unsupported vision config: {type(vision_config)}"
248 raise NotImplementedError(msg)
@@ -320,7 +320,7 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
320 video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
321 tokenizer = cached_get_tokenizer(model_config.tokenizer)
322
323 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
323 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
324 tokenizer,
325 inputs.get("prompt"),
326 inputs["prompt_token_ids"],
@@ -330,7 +330,8 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
330
331 return token_inputs(prompt_token_ids=new_token_ids,
332 prompt=new_prompt,
333 - multi_modal_data=multi_modal_data)
333 + multi_modal_data=multi_modal_data,
334 + multi_modal_placeholders={"video": ranges})
335
336 elif is_list_of(video_data, np.ndarray):
337 video_feature_size = []
vllm/model_executor/models/minicpmv.py
+3 -3
@@ -36,8 +36,8 @@ from typing_extensions import NotRequired
36
37 from vllm.attention import AttentionMetadata
38 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
39 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
40 - token_inputs)
39 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40 + InputContext, token_inputs)
41 from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization import QuantizationConfig
43 from vllm.model_executor.layers.resampler import (BaseResampler, Resampler2,
@@ -277,7 +277,7 @@ def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int,
277 seq_data = dummy_seq_data_for_minicpmv(seq_len, num_images)
278 mm_data = dummy_image_for_minicpmv(ctx, hf_config, num_images)
279
280 - return seq_data, mm_data
280 + return DummyData(seq_data, mm_data)
281
282
283 def input_processor_for_minicpmv(ctx: InputContext, inputs: DecoderOnlyInputs):
vllm/model_executor/models/mllama.py
+4 -3
@@ -36,7 +36,7 @@ from vllm.attention import Attention, AttentionMetadata, AttentionType
36 from vllm.attention.ops.paged_attn import PagedAttention
37 from vllm.config import CacheConfig, MultiModalConfig
38 from vllm.distributed import get_tensor_model_parallel_world_size
39 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
39 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
40 EncoderDecoderInputs, InputContext)
41 from vllm.logger import init_logger
42 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -176,13 +176,14 @@ def dummy_image(num_images: int, ):
176 def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
177 mm_counts: Mapping[str, int]):
178 num_images = mm_counts["image"]
179 - return dummy_decoder_seq_data(seq_len, num_images), None
179 + return DummyData(dummy_decoder_seq_data(seq_len, num_images))
180
181
182 def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
183 mm_counts: Mapping[str, int]):
184 num_images = mm_counts["image"]
185 - return dummy_encoder_seq_data(ctx, num_images), dummy_image(num_images)
185 + return DummyData(dummy_encoder_seq_data(ctx, num_images),
186 + dummy_image(num_images))
187
188
189 def _prepare_aspect_ratio_attention_mask(
vllm/model_executor/models/paligemma.py
+4 -4
@@ -7,8 +7,8 @@ from transformers import PaliGemmaConfig
7
8 from vllm.attention import AttentionMetadata
9 from vllm.config import CacheConfig, MultiModalConfig
10 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
11 - token_inputs)
10 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
11 + InputContext, token_inputs)
12 from vllm.logger import init_logger
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -58,7 +58,7 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
58 vision_config = hf_config.vision_config
59 num_images = mm_counts["image"]
60
61 - seq_data = dummy_seq_data_for_siglip(
61 + seq_data, ranges = dummy_seq_data_for_siglip(
62 vision_config,
63 seq_len,
64 num_images,
@@ -66,7 +66,7 @@ def dummy_data_for_paligemma(ctx: InputContext, seq_len: int,
66 )
67
68 mm_data = dummy_image_for_siglip(vision_config, num_images)
69 - return seq_data, mm_data
69 + return DummyData(seq_data, mm_data, ranges)
70
71
72 def input_processor_for_paligemma(ctx: InputContext,
vllm/model_executor/models/phi3v.py
+4 -4
@@ -28,8 +28,8 @@ from transformers import CLIPVisionConfig, PretrainedConfig
28 from vllm.attention import AttentionMetadata
29 from vllm.config import (CacheConfig, ModelConfig, MultiModalConfig,
30 PoolerConfig)
31 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
32 - token_inputs)
31 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
32 + InputContext, token_inputs)
33 from vllm.logger import init_logger
34 from vllm.model_executor.layers.pooler import Pooler, PoolingType
35 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -380,7 +380,7 @@ def dummy_data_for_phi3v(ctx: InputContext,
380
381 image_feature_size = get_max_phi3v_image_tokens(ctx, num_crops=num_crops)
382
383 - seq_data = dummy_seq_data_for_clip(
383 + seq_data, ranges = dummy_seq_data_for_clip(
384 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
385 seq_len,
386 num_images,
@@ -394,7 +394,7 @@ def dummy_data_for_phi3v(ctx: InputContext,
394 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
395 )
396
397 - return seq_data, mm_data
397 + return DummyData(seq_data, mm_data, ranges)
398
399
400 @lru_cache
vllm/model_executor/models/pixtral.py
+22 -12
@@ -17,8 +17,8 @@ from transformers.models.pixtral.modeling_pixtral import (
17
18 from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
20 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21 - token_inputs)
20 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 + InputContext, token_inputs)
22 from vllm.model_executor.layers.activation import get_act_fn
23 from vllm.model_executor.layers.layernorm import RMSNorm
24 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -28,7 +28,8 @@ from vllm.model_executor.models.utils import merge_multimodal_embeddings
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.multimodal import MULTIMODAL_REGISTRY
30 from vllm.multimodal.base import MultiModalInputs
31 -from vllm.multimodal.utils import cached_get_tokenizer
31 +from vllm.multimodal.utils import (cached_get_tokenizer,
32 + consecutive_placeholder_ranges)
33 from vllm.sequence import IntermediateTensors, SequenceData
34 from vllm.transformers_utils.processor import cached_get_processor
35 from vllm.utils import is_list_of
@@ -81,7 +82,12 @@ def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
82 )
83
84 mm_data = {"image": num_images * [image]}
84 - return seq_data, mm_data
85 + mm_placeholders = {
86 + "image":
87 + consecutive_placeholder_ranges(num_items=num_images,
88 + item_size=image_feature_size)
89 + }
90 + return DummyData(seq_data, mm_data, mm_placeholders)
91
92
93 def input_mapper_for_pixtral(ctx: InputContext,
@@ -630,13 +636,13 @@ def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:
636
637
638 def dummy_seq_data_for_pixtral_hf(
633 - hf_config: PixtralVisionConfig,
634 - seq_len: int,
635 - num_images: int,
636 - *,
637 - image_token_id: int,
638 - image_feature_size_override: Optional[int] = None,
639 -):
639 + hf_config: PixtralVisionConfig,
640 + seq_len: int,
641 + num_images: int,
642 + *,
643 + image_token_id: int,
644 + image_feature_size_override: Optional[int] = None,
645 + mm_key: str = "image"):
646 if image_feature_size_override is None:
647 image_feature_size = get_max_pixtral_hf_image_feature_size(hf_config)
648 else:
@@ -645,7 +651,11 @@ def dummy_seq_data_for_pixtral_hf(
651 return SequenceData.from_prompt_token_counts(
652 (image_token_id, image_feature_size * num_images),
653 (0, seq_len - image_feature_size * num_images),
648 - )
654 + ), {
655 + mm_key:
656 + consecutive_placeholder_ranges(num_items=num_images,
657 + item_size=image_feature_size)
658 + }
659
660
661 def dummy_image_for_pixtral_hf(
vllm/model_executor/models/qwen.py
+5 -5
@@ -23,8 +23,8 @@ from vllm.attention import Attention, AttentionMetadata
23 from vllm.compilation.decorators import support_torch_compile
24 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
25 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
27 - token_inputs)
26 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
27 + InputContext, token_inputs)
28 from vllm.logger import init_logger
29 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
30 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -810,7 +810,7 @@ def dummy_data_for_qwen(
810 ctx: InputContext,
811 seq_len: int,
812 mm_counts: Mapping[str, int],
813 -) -> Tuple[SequenceData, Optional[Dict]]:
813 +) -> DummyData:
814 """Build dummy data for warming up Qwen models; this will only contain text
815 matching the defaults for VLLM unless the model has a visual config.
816
@@ -829,7 +829,7 @@ def dummy_data_for_qwen(
829 if not hasattr(hf_config, "visual"):
830 seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
831 mm_data = None
832 - return seq_data, mm_data
832 + return DummyData(seq_data, mm_data)
833
834 # We have a visual component - use images to warm up
835 num_images = mm_counts["image"]
@@ -861,7 +861,7 @@ def dummy_data_for_qwen(
861 # the data will get resized and the # of tokens per image is constant
862 image = Image.new("RGB", (224, 224), color=0)
863 mm_data = {"image": image if num_images == 1 else [image] * num_images}
864 - return seq_data, mm_data
864 + return DummyData(seq_data, mm_data)
865
866
867 class QWenBaseModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
vllm/model_executor/models/qwen2_audio.py
+11 -4
@@ -31,8 +31,8 @@ from transformers import Qwen2AudioConfig, Qwen2AudioEncoder
31
32 from vllm.attention import AttentionMetadata
33 from vllm.config import CacheConfig, MultiModalConfig
34 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
35 - token_inputs)
34 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35 + InputContext, token_inputs)
36 from vllm.logger import init_logger
37 from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
@@ -44,6 +44,7 @@ from vllm.model_executor.model_loader.weight_utils import (
44 from vllm.model_executor.models.qwen2 import Qwen2Model
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
47 +from vllm.multimodal.utils import consecutive_placeholder_ranges
48 from vllm.sequence import IntermediateTensors, SequenceData
49
50 from .interfaces import SupportsMultiModal, SupportsPP
@@ -85,7 +86,8 @@ class Qwen2AudioMultiModalProjector(nn.Module):
86 def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
87 mm_counts: Mapping[str, int]):
88 num_audios = mm_counts["audio"]
88 - max_llm_audio_tokens = get_max_qwen2_audio_audio_tokens(ctx) * num_audios
89 + max_tokens_per_audio = get_max_qwen2_audio_audio_tokens(ctx)
90 + max_llm_audio_tokens = max_tokens_per_audio * num_audios
91 if seq_len - max_llm_audio_tokens - 2 < 0:
92 raise RuntimeError(
93 f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
@@ -99,7 +101,12 @@ def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
101 (0, seq_len - max_llm_audio_tokens),
102 )
103 dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
102 - return dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}
104 + return DummyData(
105 + dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}, {
106 + "audio":
107 + consecutive_placeholder_ranges(num_items=num_audios,
108 + item_size=max_tokens_per_audio)
109 + })
110
111
112 def get_processor(
vllm/model_executor/models/qwen2_vl.py
+6 -5
@@ -44,8 +44,8 @@ from vllm.attention.selector import _Backend
44 from vllm.config import CacheConfig, MultiModalConfig
45 from vllm.distributed import get_pp_group, parallel_state
46 from vllm.distributed import utils as dist_utils
47 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
48 - token_inputs)
47 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
48 + InputContext, token_inputs)
49 from vllm.logger import init_logger
50 from vllm.model_executor import SamplingMetadata
51 from vllm.model_executor.layers.activation import QuickGELU
@@ -744,9 +744,10 @@ def dummy_data_for_qwen2_vl(
744 dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
745 color=0)
746
747 - return dummy_seqdata, {
748 - "image": dummy_image if num_images == 1 else [dummy_image] * num_images
749 - }
747 + return DummyData(dummy_seqdata, {
748 + "image":
749 + dummy_image if num_images == 1 else [dummy_image] * num_images
750 + })
751
752
753 def _get_llm_num_vision_tokens(
vllm/model_executor/models/siglip.py
+17 -7
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding)
24 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25 from vllm.multimodal.utils import (cached_get_tokenizer,
26 + consecutive_placeholder_ranges,
27 repeat_and_pad_placeholder_tokens)
28 from vllm.sequence import SequenceData
29
@@ -61,6 +62,7 @@ def dummy_seq_data_for_siglip(
62 *,
63 image_token_id: int,
64 image_feature_size_override: Optional[int] = None,
65 + mm_key: str = "image",
66 ):
67 if image_feature_size_override is None:
68 image_feature_size = get_siglip_image_feature_size(hf_config)
@@ -70,7 +72,11 @@ def dummy_seq_data_for_siglip(
72 return SequenceData.from_prompt_token_counts(
73 (image_token_id, image_feature_size * num_images),
74 (0, seq_len - image_feature_size * num_images),
73 - )
75 + ), {
76 + mm_key:
77 + consecutive_placeholder_ranges(num_items=num_images,
78 + item_size=image_feature_size)
79 + }
80
81
82 def dummy_image_for_siglip(
@@ -122,6 +128,11 @@ def input_processor_for_siglip(
128 if multi_modal_data is None or "image" not in multi_modal_data:
129 return inputs
130
131 + if "multi_modal_placeholders" in inputs and "image" in inputs[
132 + "multi_modal_placeholders"]:
133 + # The inputs already have placeholders.
134 + return inputs
135 +
136 tokenizer = cached_get_tokenizer(model_config.tokenizer)
137
138 if image_feature_size_override is None:
@@ -135,7 +146,7 @@ def input_processor_for_siglip(
146 else:
147 image_feature_size = image_feature_size_override
148
138 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
149 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
150 tokenizer,
151 inputs.get("prompt"),
152 inputs["prompt_token_ids"],
@@ -144,11 +155,10 @@ def input_processor_for_siglip(
155 )
156
157 # NOTE: Create a defensive copy of the original inputs
147 - return token_inputs(
148 - prompt_token_ids=new_token_ids,
149 - prompt=new_prompt,
150 - multi_modal_data=multi_modal_data,
151 - )
158 + return token_inputs(prompt_token_ids=new_token_ids,
159 + prompt=new_prompt,
160 + multi_modal_data=multi_modal_data,
161 + multi_modal_placeholders={"image": ranges})
162
163
164 # Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
vllm/model_executor/models/ultravox.py
+32 -28
@@ -2,7 +2,6 @@
2 """PyTorch Ultravox model."""
3
4 import math
5 -from array import array
5 from functools import cached_property, lru_cache
6 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
7 TypedDict, Union, cast)
@@ -17,27 +16,27 @@ from transformers.models.whisper.modeling_whisper import WhisperEncoder
16
17 from vllm.attention import AttentionMetadata
18 from vllm.config import CacheConfig, MultiModalConfig
20 -from vllm.inputs import INPUT_REGISTRY
21 -from vllm.inputs.data import DecoderOnlyInputs, token_inputs
22 -from vllm.inputs.registry import InputContext
19 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20 + InputContext, token_inputs)
21 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
22 from vllm.model_executor.layers.layernorm import RMSNorm
23 from vllm.model_executor.layers.quantization import QuantizationConfig
24 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25 from vllm.model_executor.model_loader.loader import DefaultModelLoader
26 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 -from vllm.multimodal import MULTIMODAL_REGISTRY
30 -from vllm.multimodal.base import MultiModalInputs, NestedTensors
27 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
28 + NestedTensors)
29 from vllm.multimodal.utils import (cached_get_tokenizer,
30 + consecutive_placeholder_ranges,
31 repeat_and_pad_placeholder_tokens)
33 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
34 - SequenceData)
32 +from vllm.sequence import IntermediateTensors, SequenceData
33 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34 from vllm.utils import is_list_of
35
36 from .interfaces import SupportsMultiModal, SupportsPP
37 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
40 - init_vllm_registered_model, merge_multimodal_embeddings)
38 + init_vllm_registered_model,
39 + merge_multimodal_embeddings_from_map)
40
41 _AUDIO_PLACEHOLDER_TOKEN = 128002
42 _AUDIO_TOKENS_PER_SECOND = 6.25
@@ -46,13 +45,13 @@ _AUDIO_TOKENS_PER_SECOND = 6.25
45 class UltravoxAudioFeatureInputs(TypedDict):
46 type: Literal["audio_features"]
47 data: NestedTensors
49 - """Shape: `(batch_size, num_audios, 80, M)"""
48 + """Shape: `(batch_size, num_audios, 80, M)`"""
49
50
51 class UltravoxAudioEmbeddingInputs(TypedDict):
52 type: Literal["audio_embeds"]
53 data: NestedTensors
55 - """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
54 + """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)`"""
55
56
57 UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
@@ -79,17 +78,16 @@ def dummy_seq_data_for_ultravox(
78 seq_len: int,
79 audio_count: int,
80 ):
82 - audio_placeholder = array(
83 - VLLM_TOKEN_ID_ARRAY_TYPE,
84 - [_AUDIO_PLACEHOLDER_TOKEN]) * get_ultravox_max_audio_tokens(ctx)
81 + audio_length = min(get_ultravox_max_audio_tokens(ctx),
82 + seq_len // audio_count)
83
86 - # Add a separator between each chunk.
87 - audio_token_ids = (audio_placeholder +
88 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [0])) * audio_count
89 - other_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
90 - [0]) * (seq_len - len(audio_token_ids))
91 -
92 - return SequenceData(audio_token_ids + other_token_ids)
84 + return SequenceData.from_prompt_token_counts(
85 + (_AUDIO_PLACEHOLDER_TOKEN, audio_length * audio_count),
86 + (0, seq_len - audio_length * audio_count)), {
87 + "audio":
88 + consecutive_placeholder_ranges(num_items=audio_count,
89 + item_size=audio_length)
90 + }
91
92
93 def dummy_audio_for_ultravox(
@@ -107,10 +105,10 @@ def dummy_data_for_ultravox(
105 mm_counts: Mapping[str, int],
106 ):
107 audio_count = mm_counts["audio"]
110 - seq_data = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
108 + seq_data, ranges = dummy_seq_data_for_ultravox(ctx, seq_len, audio_count)
109 mm_dict = dummy_audio_for_ultravox(ctx, audio_count)
110
113 - return (seq_data, mm_dict)
111 + return DummyData(seq_data, mm_dict, ranges)
112
113
114 def input_mapper_for_ultravox(ctx: InputContext, data: object):
@@ -164,6 +162,11 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
162 if multi_modal_data is None or "audio" not in multi_modal_data:
163 return inputs
164
165 + if "multi_modal_placeholders" in inputs and "audio" in inputs[
166 + "multi_modal_placeholders"]:
167 + # The inputs already have placeholders.
168 + return inputs
169 +
170 feature_extractor = whisper_feature_extractor(ctx)
171 audios = multi_modal_data["audio"]
172 if not isinstance(audios, list):
@@ -197,7 +200,7 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
200
201 tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
202
200 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
203 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
204 tokenizer,
205 inputs.get("prompt"),
206 inputs["prompt_token_ids"],
@@ -208,7 +211,8 @@ def input_processor_for_ultravox(ctx: InputContext, inputs: DecoderOnlyInputs):
211 # NOTE: Create a defensive copy of the original inputs
212 return token_inputs(prompt_token_ids=new_token_ids,
213 prompt=new_prompt,
211 - multi_modal_data=multi_modal_data)
214 + multi_modal_data=multi_modal_data,
215 + multi_modal_placeholders={"audio": ranges})
216
217
218 class StackAudioFrames(nn.Module):
@@ -472,9 +476,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
476 inputs_embeds = self.language_model.model.get_input_embeddings(
477 input_ids)
478
475 - inputs_embeds = merge_multimodal_embeddings(
476 - input_ids, inputs_embeds, audio_embeddings,
477 - _AUDIO_PLACEHOLDER_TOKEN)
479 + merge_multimodal_embeddings_from_map(
480 + inputs_embeds, audio_embeddings,
481 + attn_metadata.multi_modal_placeholder_index_maps["audio"])
482 input_ids = None
483 else:
484 inputs_embeds = None
vllm/model_executor/models/utils.py
+17 -1
@@ -18,7 +18,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.model_loader.loader import build_model
19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.model_executor.models import ModelRegistry
21 -from vllm.multimodal.base import NestedTensors
21 +from vllm.multimodal.base import MultiModalPlaceholderMap, NestedTensors
22 from vllm.platforms import current_platform
23 from vllm.sequence import IntermediateTensors
24 from vllm.utils import is_pin_memory_available
@@ -326,6 +326,22 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
326 _embedding_count_expression(inner) for inner in embeddings)
327
328
329 +def merge_multimodal_embeddings_from_map(
330 + inputs_embeds: torch.Tensor, multimodal_embeddings: NestedTensors,
331 + placeholder_map: MultiModalPlaceholderMap.IndexMap) -> torch.Tensor:
332 + """
333 + Merge ``multimodal_embeddings`` into ``inputs_embeds`` using the provided
334 + placeholder map .
335 +
336 + Note:
337 + This updates ``inputs_embeds`` in place.
338 + """
339 + flattened_embeddings = _flatten_embeddings(multimodal_embeddings)
340 + inputs_embeds[placeholder_map.dest] = flattened_embeddings[
341 + placeholder_map.src]
342 + return inputs_embeds
343 +
344 +
345 def _merge_multimodal_embeddings(
346 inputs_embeds: torch.Tensor,
347 is_multimodal: torch.Tensor,
vllm/multimodal/__init__.py
+5 -2
@@ -1,6 +1,7 @@
1 from .base import (BatchedTensorInputs, MultiModalDataBuiltins,
2 - MultiModalDataDict, MultiModalInputs, MultiModalPlugin,
3 - NestedTensors)
2 + MultiModalDataDict, MultiModalInputs,
3 + MultiModalPlaceholderDict, MultiModalPlaceholderMap,
4 + MultiModalPlugin, NestedTensors)
5 from .registry import MultiModalRegistry
6
7 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -17,6 +18,8 @@ __all__ = [
18 "MultiModalDataBuiltins",
19 "MultiModalDataDict",
20 "MultiModalInputs",
21 + "MultiModalPlaceholderDict",
22 + "MultiModalPlaceholderMap",
23 "MultiModalPlugin",
24 "NestedTensors",
25 "MULTIMODAL_REGISTRY",
vllm/multimodal/base.py
+209 -5
@@ -1,8 +1,9 @@
1 import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 -from typing import (Any, Callable, Dict, List, Mapping, Optional, Tuple, Type,
5 - TypedDict, TypeVar, Union, cast, final)
4 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
5 + NamedTuple, Optional, Tuple, Type, TypedDict, TypeVar,
6 + Union, cast, final)
7
8 import numpy as np
9 import torch
@@ -11,12 +12,15 @@ from PIL import Image
12 from torch import nn
13 from typing_extensions import TypeAlias
14
14 -from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
17 from vllm.utils import (JSONTree, get_allowed_kwarg_only_overrides, is_list_of,
18 json_map_leaves, resolve_mm_processor_kwargs)
19
20 +if TYPE_CHECKING:
21 + from vllm.config import ModelConfig
22 + from vllm.sequence import SequenceGroupMetadata
23 +
24 logger = init_logger(__name__)
25
26 NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor]
@@ -151,6 +155,30 @@ Note:
155 Read more on that :ref:`here <adding_multimodal_plugin>`.
156 """
157
158 +
159 +class PlaceholderRange(TypedDict):
160 + """
161 + Placeholder location information for multi-modal data.
162 +
163 + For example:
164 + Prompt: AAAA BBBB What is in these images?
165 + Images A and B will have:
166 + A: { "offset": 0, "length": 4 }
167 + B: { "offset": 5, "length": 4 }
168 + """
169 +
170 + offset: int
171 + """The start index of the placeholder in the prompt."""
172 +
173 + length: int
174 + """The length of the placeholder."""
175 +
176 +
177 +MultiModalPlaceholderDict = Mapping[str, List[PlaceholderRange]]
178 +"""
179 +A dictionary containing placeholder ranges.
180 +"""
181 +
182 MultiModalInputMapper = Callable[[InputContext, MultiModalData[object]],
183 MultiModalInputs]
184 """
@@ -243,7 +271,7 @@ class MultiModalPlugin(ABC):
271
272 return wrapper
273
246 - def map_input(self, model_config: ModelConfig,
274 + def map_input(self, model_config: "ModelConfig",
275 data: MultiModalData[object],
276 mm_processor_kwargs: Dict[str, Any]) -> MultiModalInputs:
277 """
@@ -332,7 +360,7 @@ class MultiModalPlugin(ABC):
360
361 return wrapper
362
335 - def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
363 + def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
364 """
365 Get the maximum number of multi-modal tokens
366 for profiling the memory usage of a model.
@@ -366,3 +394,179 @@ class MultiModalPlugin(ABC):
394 self._validate_max_multimodal_tokens(max_mm_tokens)
395
396 return max_mm_tokens
397 +
398 +
399 +class MultiModalPlaceholderMap:
400 + """
401 + Relates multi-modal embeddings to their corresponding placeholders.
402 + """
403 +
404 + class IndexMap(NamedTuple):
405 + src: List[int]
406 + dest: List[int]
407 +
408 + src_ranges: List[range]
409 + """
410 + The indices of the multi-modal embeddings that will replace the
411 + corresponding placeholder embeddings pointed to by ``dest_ranges``.
412 + """
413 +
414 + src_len: int
415 + """
416 + The total number of flattened multi-modal embeddings.
417 + """
418 +
419 + dest_ranges: List[range]
420 + """
421 + The indices of the placeholder embeddings that will be replaced by the
422 + multimodal embeddings.
423 + """
424 +
425 + dest_len: int
426 + """
427 + The total number of embeddings in the destination tensor.
428 + """
429 +
430 + def __init__(self):
431 + self.src_ranges = []
432 + self.src_len = 0
433 + self.dest_ranges = []
434 + self.dest_len = 0
435 +
436 + @classmethod
437 + def from_seq_group(
438 + cls, seq_group: "SequenceGroupMetadata", positions: range
439 + ) -> Tuple[Optional[MultiModalDataDict], Dict[str,
440 + "MultiModalPlaceholderMap"]]:
441 + """
442 + Returns the multi-modal items that intersect with the portion of a
443 + prompt (``seq_group``) represented by ``positions``, as well as a
444 + ``MultiModalPlaceholderMap`` that relates the multi-modal embedding
445 + vectors to their corresponding placeholders.
446 +
447 + Consider the following scenarios:
448 +
449 + Prompt: |AAAA BBBB What's in these images?|
450 + Positions: |.................................|
451 +
452 + images = [A, B]
453 + src_ranges = [(0, 4), (4, 8)]
454 + dest_ranges = [(0, 4), (5, 9)]
455 +
456 + Prompt: |AAAA BBBB What's in these images?|
457 + Positions: | ..... |
458 +
459 + images = [A, B]
460 + src_ranges = [(2, 4), (4, 6)]
461 + dest_ranges = [(0, 2), (3, 5)]
462 +
463 + Prompt: |AAAA BBBB What's in these images?|
464 + Positions: | ......... |
465 +
466 + images = [B]
467 + src_ranges = [(0, 4)]
468 + dest_ranges = [(0, 4)]
469 +
470 + Prompt: |AAAA BBBB What's in these images?|
471 + Positions: | .......................|
472 +
473 + images = []
474 + src_ranges = []
475 + dest_ranges = []
476 + """
477 + if (not seq_group.multi_modal_data
478 + or not seq_group.multi_modal_placeholders):
479 + return seq_group.multi_modal_data, {}
480 +
481 + mm_data = {**seq_group.multi_modal_data}
482 + placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
483 + MultiModalPlaceholderMap)
484 +
485 + for modality, placeholders in seq_group.multi_modal_placeholders.items(
486 + ):
487 + mm_items = mm_data.pop(modality)
488 + if not isinstance(mm_items, list):
489 + mm_items = [mm_items]
490 +
491 + if positions:
492 + intersecting_items = placeholder_maps[
493 + modality].append_items_from_seq_group(
494 + positions, mm_items, placeholders)
495 +
496 + if intersecting_items:
497 + mm_data[modality] = intersecting_items
498 +
499 + return mm_data, placeholder_maps
500 +
501 + def append_items_from_seq_group(
502 + self, positions: range, multi_modal_items: List[_T],
503 + multi_modal_placeholders: List[PlaceholderRange]) -> List[_T]:
504 + """
505 + Adds the multi-modal items that intersect ```positions`` to this
506 + placeholder map and returns the intersecting items.
507 + """
508 + intersecting_items = []
509 +
510 + if len(multi_modal_items) != len(multi_modal_placeholders):
511 + raise ValueError(
512 + "Multi-modal placeholders and items must have the same length."
513 + )
514 + for placeholder_dict, mm_item in zip(multi_modal_placeholders,
515 + multi_modal_items):
516 + placeholder = range(
517 + placeholder_dict["offset"],
518 + placeholder_dict["offset"] + placeholder_dict["length"])
519 + intersection = range(max(positions.start, placeholder.start),
520 + min(positions.stop, placeholder.stop))
521 +
522 + if not intersection:
523 + # Skip this multi-modal item.
524 + continue
525 +
526 + token_embedding_range = range(intersection.start - positions.start,
527 + intersection.stop - positions.start)
528 +
529 + multimodal_embedding_range = range(
530 + intersection.start - placeholder.start + self.src_len,
531 + intersection.stop - placeholder.start + self.src_len)
532 +
533 + intersecting_items.append(mm_item)
534 + self.dest_ranges.append(token_embedding_range)
535 + self.src_ranges.append(multimodal_embedding_range)
536 + self.src_len += len(placeholder)
537 +
538 + self.dest_len += len(positions)
539 + return intersecting_items
540 +
541 + def extend(self, other: "MultiModalPlaceholderMap"):
542 + """
543 + Adds the placeholders from another ``MultiModalPlaceholderMap`` to this
544 + instance based on the source and destination tensors being
545 + concatenated.
546 + """
547 +
548 + self.src_ranges.extend(
549 + range(self.src_len + r.start, self.src_len + r.stop)
550 + for r in other.src_ranges)
551 + self.src_len += other.src_len
552 + self.dest_ranges.extend(
553 + range(self.dest_len + r.start, self.dest_len + r.stop)
554 + for r in other.dest_ranges)
555 + self.dest_len += other.dest_len
556 +
557 + def index_map(self) -> "IndexMap":
558 + """
559 + Finalizes the placeholder map into lists of indices that can be used to
560 + index the source and destination tensors.
561 + """
562 +
563 + src_indices = [i for r in self.src_ranges for i in r]
564 + dest_indices = [i for r in self.dest_ranges for i in r]
565 +
566 + if len(src_indices) != len(dest_indices):
567 + raise ValueError(
568 + f"The number of source ({len(src_indices)}) and destination "
569 + f"indices ({len(dest_indices)}) must be the same.")
570 +
571 + return MultiModalPlaceholderMap.IndexMap(src=src_indices,
572 + dest=dest_indices)
vllm/multimodal/image.py
+5 -3
@@ -1,11 +1,10 @@
1 from functools import lru_cache
2 -from typing import Any, Dict, Optional
2 +from typing import TYPE_CHECKING, Any, Dict, Optional
3
4 import torch
5 from PIL import Image
6 from transformers.image_processing_base import BatchFeature
7
8 -from vllm.config import ModelConfig
8 from vllm.inputs.registry import InputContext
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.processor import get_image_processor
@@ -13,6 +12,9 @@ from vllm.utils import is_list_of
12
13 from .base import MultiModalData, MultiModalInputs, MultiModalPlugin
14
15 +if TYPE_CHECKING:
16 + from vllm.config import ModelConfig
17 +
18 logger = init_logger(__name__)
19
20 cached_get_image_processor = lru_cache(get_image_processor)
@@ -26,7 +28,7 @@ class ImagePlugin(MultiModalPlugin):
28
29 def _get_hf_image_processor(
30 self,
29 - model_config: ModelConfig,
31 + model_config: "ModelConfig",
32 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
33 ):
34 if mm_processor_kwargs is None:
vllm/multimodal/registry.py
+10 -8
@@ -1,8 +1,7 @@
1 import functools
2 from collections import UserDict
3 -from typing import Any, Dict, Mapping, Optional, Sequence
3 +from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Sequence
4
5 -from vllm.config import ModelConfig
5 from vllm.logger import init_logger
6
7 from .audio import AudioPlugin
@@ -11,6 +10,9 @@ from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10 from .image import ImagePlugin
11 from .video import VideoPlugin
12
13 +if TYPE_CHECKING:
14 + from vllm.config import ModelConfig
15 +
16 logger = init_logger(__name__)
17
18
@@ -20,7 +22,7 @@ class _MultiModalLimits(UserDict):
22 when attempting to access a model that does not exist.
23 """
24
23 - def __getitem__(self, key: ModelConfig) -> Dict[str, int]:
25 + def __getitem__(self, key: "ModelConfig") -> Dict[str, int]:
26 try:
27 return super().__getitem__(key)
28 except KeyError as exc:
@@ -98,7 +100,7 @@ class MultiModalRegistry:
100
101 def map_input(
102 self,
101 - model_config: ModelConfig,
103 + model_config: "ModelConfig",
104 data: MultiModalDataDict,
105 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
106 ) -> MultiModalInputs:
@@ -139,7 +141,7 @@ class MultiModalRegistry:
141
142 return MultiModalInputs(merged_dict)
143
142 - def create_input_mapper(self, model_config: ModelConfig):
144 + def create_input_mapper(self, model_config: "ModelConfig"):
145 """
146 Create an input mapper (see :meth:`map_input`) for a specific model.
147 """
@@ -177,7 +179,7 @@ class MultiModalRegistry:
179 """
180 return self.register_max_multimodal_tokens("image", max_mm_tokens)
181
180 - def get_max_multimodal_tokens(self, model_config: ModelConfig) -> int:
182 + def get_max_multimodal_tokens(self, model_config: "ModelConfig") -> int:
183 """
184 Get the maximum number of multi-modal tokens
185 for profiling the memory usage of a model.
@@ -195,7 +197,7 @@ class MultiModalRegistry:
197
198 def init_mm_limits_per_prompt(
199 self,
198 - model_config: ModelConfig,
200 + model_config: "ModelConfig",
201 ) -> None:
202 """
203 Initialize the maximum number of multi-modal input instances for each
@@ -231,7 +233,7 @@ class MultiModalRegistry:
233
234 def get_mm_limits_per_prompt(
235 self,
234 - model_config: ModelConfig,
236 + model_config: "ModelConfig",
237 ) -> Mapping[str, int]:
238 """
239 Get the maximum number of multi-modal input instances for each modality
vllm/multimodal/utils.py
+18 -3
@@ -10,7 +10,7 @@ from PIL import Image
10 from vllm.connections import global_http_connection
11 from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
12 from vllm.logger import init_logger
13 -from vllm.multimodal.base import MultiModalDataDict
13 +from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
14 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
15
16 logger = init_logger(__name__)
@@ -258,7 +258,7 @@ def repeat_and_pad_placeholder_tokens(
258 repeat_count: Union[int, List[int]],
259 pad_token_left: Optional[int] = None,
260 pad_token_right: Optional[int] = None,
261 -) -> Tuple[Optional[str], List[int]]:
261 +) -> Tuple[Optional[str], List[int], List[PlaceholderRange]]:
262 if isinstance(repeat_count, int):
263 repeat_count = [repeat_count]
264
@@ -301,6 +301,7 @@ def repeat_and_pad_placeholder_tokens(
301 new_prompt += prompt_parts[-1]
302
303 new_token_ids: List[int] = []
304 + placeholder_ranges: List[PlaceholderRange] = []
305 placeholder_token_idx = 0
306 for i, token in enumerate(prompt_token_ids):
307 if token == placeholder_token_id:
@@ -310,6 +311,10 @@ def repeat_and_pad_placeholder_tokens(
311 pad_token_left=pad_token_left,
312 pad_token_right=pad_token_right,
313 )
314 + placeholder_ranges.append({
315 + "offset": len(new_token_ids),
316 + "length": len(replacement_ids)
317 + })
318 new_token_ids.extend(replacement_ids)
319 placeholder_token_idx += 1
320
@@ -320,4 +325,14 @@ def repeat_and_pad_placeholder_tokens(
325 else:
326 new_token_ids.append(token)
327
323 - return new_prompt, new_token_ids
328 + return new_prompt, new_token_ids, placeholder_ranges
329 +
330 +
331 +def consecutive_placeholder_ranges(num_items: int,
332 + item_size: int) -> List[PlaceholderRange]:
333 + """Returns a list of consecutive PlaceholderRanges of a fixed size"""
334 +
335 + return [
336 + PlaceholderRange(offset=i * item_size, length=item_size)
337 + for i in range(num_items)
338 + ]
vllm/multimodal/video.py
+9 -5
@@ -1,18 +1,19 @@
1 from functools import lru_cache
2 -from typing import Any, Dict, List, Optional, Union
2 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
3
4 import numpy as np
5
6 -from vllm.config import ModelConfig
6 from vllm.inputs.registry import InputContext
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.processor import get_video_processor
9 from vllm.transformers_utils.tokenizer import get_tokenizer
11 -from vllm.utils import is_list_of
10
11 from .base import MultiModalData, MultiModalInputs
12 from .image import ImagePlugin
13
14 +if TYPE_CHECKING:
15 + from vllm.config import ModelConfig
16 +
17 logger = init_logger(__name__)
18
19 cached_get_video_processor = lru_cache(get_video_processor)
@@ -38,7 +39,7 @@ class VideoPlugin(ImagePlugin):
39
40 def _get_hf_video_processor(
41 self,
41 - model_config: ModelConfig,
42 + model_config: "ModelConfig",
43 mm_processor_kwargs: Optional[Dict[str, Any]] = None,
44 ):
45 if mm_processor_kwargs is None:
@@ -56,7 +57,10 @@ class VideoPlugin(ImagePlugin):
57 ) -> MultiModalInputs:
58 model_config = ctx.model_config
59
59 - if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
60 + if isinstance(data, list) and len(data) == 1:
61 + data = data[0]
62 +
63 + if isinstance(data, np.ndarray):
64 video_processor = self._get_hf_video_processor(
65 model_config,
66 mm_processor_kwargs,
vllm/sequence.py
+13 -4
@@ -15,13 +15,13 @@ import torch
15
16 from vllm.inputs.parse import is_encoder_decoder_inputs
17 from vllm.lora.request import LoRARequest
18 +from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
19 from vllm.pooling_params import PoolingParams
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sampling_params import RequestOutputKind, SamplingParams
22
23 if TYPE_CHECKING:
24 from vllm.inputs import SingletonInputs
24 - from vllm.multimodal.base import MultiModalDataDict
25
26 VLLM_TOKEN_ID_ARRAY_TYPE = "l"
27
@@ -485,7 +485,7 @@ class Sequence:
485 return cast(List[int], self.inputs.get(prompt_token_ids_key))
486
487 @property
488 - def multi_modal_data(self) -> "MultiModalDataDict":
488 + def multi_modal_data(self) -> MultiModalDataDict:
489 inputs = self.inputs
490
491 if (inputs.get("multi_modal_data")
@@ -495,11 +495,15 @@ class Sequence:
495 )
496
497 return cast(
498 - "MultiModalDataDict",
498 + MultiModalDataDict,
499 (inputs.get("multi_modal_data")
500 or inputs.get("encoder_multi_modal_data") or {}),
501 )
502
503 + @property
504 + def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
505 + return self.inputs.get("multi_modal_placeholders") or {}
506 +
507 @property
508 def mm_processor_kwargs(self) -> Dict[str, Any]:
509 return self.inputs.get("mm_processor_kwargs") or {}
@@ -728,9 +732,13 @@ class SequenceGroup:
732 if self.encoder_seq is not None else None)
733
734 @property
731 - def multi_modal_data(self) -> "MultiModalDataDict":
735 + def multi_modal_data(self) -> MultiModalDataDict:
736 return self.first_seq.multi_modal_data
737
738 + @property
739 + def multi_modal_placeholders(self) -> MultiModalPlaceholderDict:
740 + return self.first_seq.multi_modal_placeholders
741 +
742 @property
743 def mm_processor_kwargs(self) -> Dict[str, Any]:
744 return self.first_seq.mm_processor_kwargs
@@ -946,6 +954,7 @@ class SequenceGroupMetadata(
954 # "MultiModalDataDict" types. We have to use Any due to msgspec
955 # doesn't allow to have union of 2 different dicts.
956 multi_modal_data: Optional[Any] = None
957 + multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
958 mm_processor_kwargs: Optional[Dict[str, Any]] = None
959 encoder_seq_data: Optional[SequenceData] = None
960 cross_block_table: Optional[List[int]] = None
vllm/worker/cpu_model_runner.py
+31 -7
@@ -1,5 +1,6 @@
1 import dataclasses
2 import weakref
3 +from collections import defaultdict
4 from dataclasses import dataclass
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
6
@@ -16,7 +17,7 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
17 from vllm.model_executor.layers.sampler import SamplerOutput
18 from vllm.model_executor.model_loader import get_model
19 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 - MultiModalInputs)
20 + MultiModalInputs, MultiModalPlaceholderMap)
21 from vllm.sequence import (IntermediateTensors, SequenceData,
22 SequenceGroupMetadata)
23 from vllm.transformers_utils.config import uses_mrope
@@ -148,9 +149,18 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
149 query_lens=seq_lens,
150 )
151
151 - def _compute_multi_modal_input(self, seq_data: SequenceData, mm_data,
152 - computed_len: int,
152 + def _compute_multi_modal_input(self, seq_group: SequenceGroupMetadata,
153 + seq_data: SequenceData, computed_len: int,
154 mm_processor_kwargs: Dict[str, Any]):
155 +
156 + # NOTE: mm_data only includes the subset of multi-modal items that
157 + # intersect with the current prefill positions.
158 + mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
159 + seq_group, range(computed_len, len(seq_data.get_token_ids())))
160 +
161 + if not mm_data:
162 + return
163 +
164 mm_kwargs = self.multi_modal_input_mapper(mm_data, mm_processor_kwargs)
165
166 # special processing for mrope position deltas.
@@ -179,7 +189,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
189 context_len=computed_len,
190 )
191 seq_data.mrope_position_delta = mrope_position_delta
182 - return mm_kwargs, mrope_positions
192 + return mm_kwargs, placeholder_maps, mrope_positions
193
194 def _prepare_prompt(
195 self,
@@ -194,6 +204,9 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
204 slot_mapping: List[int] = []
205 seq_lens: List[int] = []
206 multi_modal_inputs_list: List[MultiModalInputs] = []
207 + multi_modal_placeholder_maps: Dict[
208 + str,
209 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
210
211 for seq_group_metadata in seq_group_metadata_list:
212 assert seq_group_metadata.is_prompt
@@ -210,11 +223,15 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
223 input_tokens.extend(prompt_tokens) # Token ids
224
225 mrope_positions = None
213 - if (mm_data := seq_group_metadata.multi_modal_data):
214 - mm_kwargs, mrope_positions = self._compute_multi_modal_input(
215 - seq_data, mm_data, computed_len,
226 + if seq_group_metadata.multi_modal_data:
227 + mm_kwargs, placeholder_maps, mrope_positions = self \
228 + ._compute_multi_modal_input(
229 + seq_group_metadata, seq_data, computed_len,
230 seq_group_metadata.mm_processor_kwargs)
231 multi_modal_inputs_list.append(mm_kwargs)
232 + for modality, placeholder_map in placeholder_maps.items():
233 + multi_modal_placeholder_maps[modality].extend(
234 + placeholder_map)
235
236 # Token position ids
237 # NOTE(woosuk): Here we assume that the first token in the prompt
@@ -264,6 +281,11 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
281 slot_mapping = torch.tensor(slot_mapping,
282 dtype=torch.long,
283 device=self.device) # type: ignore
284 + placeholder_index_maps = {
285 + modality: placeholder_map.index_map()
286 + for modality, placeholder_map in
287 + multi_modal_placeholder_maps.items()
288 + }
289
290 attn_metadata = self.attn_backend.make_metadata(
291 is_prompt=True,
@@ -275,6 +297,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
297 num_decode_tokens=0,
298 block_tables=torch.tensor([]),
299 slot_mapping=slot_mapping,
300 + multi_modal_placeholder_index_maps=placeholder_index_maps,
301 )
302
303 multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
@@ -366,6 +389,7 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
389 attn_metadata = self.attn_backend.make_metadata(
390 is_prompt=False,
391 slot_mapping=slot_mapping,
392 + multi_modal_placeholder_index_maps=None,
393 seq_lens=seq_lens,
394 seq_lens_tensor=seq_lens_tensor,
395 max_decode_seq_len=max_decode_seq_len,
vllm/worker/enc_dec_model_runner.py
+17 -13
@@ -306,13 +306,12 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
306 (group_id < max_num_batched_tokens % max_num_seqs))
307 batch_size += seq_len
308
309 - decoder_seq_data, decoder_dummy_multi_modal_data \
310 - = self.input_registry.dummy_data_for_profiling(
311 - self.model_config,
309 + decoder_dummy_data = self.input_registry \
310 + .dummy_data_for_profiling(self.model_config,
311 seq_len,
312 self.mm_registry,
313 is_encoder_data=False)
315 - encoder_seq_data, encoder_dummy_multi_modal_data \
314 + encoder_dummy_data \
315 = self.input_registry.dummy_data_for_profiling(
316 self.model_config,
317 seq_len,
@@ -320,26 +319,31 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
319 is_encoder_data=True)
320
321 # Having more tokens is over-conservative but otherwise fine
323 - assert len(decoder_seq_data.prompt_token_ids) >= seq_len, (
322 + assert len(
323 + decoder_dummy_data.seq_data.prompt_token_ids
324 + ) >= seq_len, (
325 f"Expected at least {seq_len} dummy tokens for profiling, "
325 - f"but got: {len(decoder_seq_data.prompt_token_ids)}")
326 + f"but got: {len(decoder_dummy_data.seq_data.prompt_token_ids)}"
327 + )
328
327 - assert decoder_dummy_multi_modal_data is None or \
328 - encoder_dummy_multi_modal_data is None, (
329 + assert decoder_dummy_data.multi_modal_data is None or \
330 + encoder_dummy_data.multi_modal_data is None, (
331 "Multi-modal data can't be provided in both encoder and decoder"
332 )
333
334 seq = SequenceGroupMetadata(
335 request_id=str(group_id),
336 is_prompt=True,
335 - seq_data={group_id: decoder_seq_data},
337 + seq_data={group_id: decoder_dummy_data.seq_data},
338 sampling_params=sampling_params,
339 block_tables=None,
338 - encoder_seq_data=encoder_seq_data,
340 + encoder_seq_data=encoder_dummy_data.seq_data,
341 cross_block_table=None,
340 - multi_modal_data=decoder_dummy_multi_modal_data
341 - or encoder_dummy_multi_modal_data,
342 - )
342 + multi_modal_data=decoder_dummy_data.multi_modal_data
343 + or encoder_dummy_data.multi_modal_data,
344 + multi_modal_placeholders=decoder_dummy_data.
345 + multi_modal_placeholders
346 + or encoder_dummy_data.multi_modal_placeholders)
347 seqs.append(seq)
348
349 # Run the model with the dummy inputs.
vllm/worker/model_runner.py
+16 -5
@@ -40,7 +40,8 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
40 from vllm.model_executor.models import supports_lora, supports_multimodal
41 from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
42 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
43 - MultiModalInputs, MultiModalRegistry)
43 + MultiModalInputs, MultiModalPlaceholderMap,
44 + MultiModalRegistry)
45 from vllm.platforms import current_platform
46 from vllm.prompt_adapter.layers import PromptAdapterMapping
47 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -242,6 +243,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
243
244 # Multi-modal inputs.
245 multi_modal_inputs: Optional[MultiModalInputs] = None,
246 + multi_modal_placeholder_maps: Optional[Dict[
247 + str, MultiModalPlaceholderMap]] = None,
248
249 # Whether the prefix cache is hit (prefill only).
250 prefix_cache_hit: bool = False,
@@ -361,6 +364,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
364
365 self.prompt_adapter_request = prompt_adapter_request
366 self.multi_modal_inputs = multi_modal_inputs
367 + self.multi_modal_placeholder_maps = multi_modal_placeholder_maps
368 self.prefix_cache_hit = prefix_cache_hit
369
370 self.n_seqs = len(self.seq_ids)
@@ -635,7 +639,12 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
639 def _compute_multi_modal_input(self, inter_data: InterDataForSeqGroup,
640 seq_group_metadata: SequenceGroupMetadata):
641 """If multi-modal data is given, add it to the input."""
638 - mm_data = seq_group_metadata.multi_modal_data
642 + # NOTE: mm_data only includes the subset of multi-modal items that
643 + # intersect with the current prefill positions.
644 + positions = inter_data.input_positions[0]
645 + mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
646 + seq_group_metadata,
647 + range(positions[0], positions[0] + len(positions)))
648 if not mm_data:
649 return
650
@@ -643,6 +652,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
652 mm_data,
653 mm_processor_kwargs=seq_group_metadata.mm_processor_kwargs)
654 inter_data.multi_modal_inputs = mm_kwargs
655 + inter_data.multi_modal_placeholder_maps = placeholder_maps
656
657 # special processing for mrope position deltas.
658 if self.runner.model_is_mrope:
@@ -1255,7 +1265,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1265 (group_id < max_num_batched_tokens % max_num_seqs))
1266 batch_size += seq_len
1267
1258 - seq_data, dummy_multi_modal_data = self.input_registry \
1268 + dummy_data = self.input_registry \
1269 .dummy_data_for_profiling(self.model_config,
1270 seq_len,
1271 self.mm_registry)
@@ -1263,12 +1273,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1273 seq = SequenceGroupMetadata(
1274 request_id=str(group_id),
1275 is_prompt=True,
1266 - seq_data={group_id: seq_data},
1276 + seq_data={group_id: dummy_data.seq_data},
1277 sampling_params=sampling_params,
1278 block_tables=None,
1279 lora_request=dummy_lora_requests_per_seq[group_id]
1280 if dummy_lora_requests_per_seq else None,
1271 - multi_modal_data=dummy_multi_modal_data,
1281 + multi_modal_data=dummy_data.multi_modal_data,
1282 + multi_modal_placeholders=dummy_data.multi_modal_placeholders,
1283 )
1284 seqs.append(seq)
1285
vllm/worker/model_runner_base.py
+2 -3
@@ -46,9 +46,8 @@ def _init_attn_metadata_from_tensor_dict(
46 # Extract the fields used to create AttentionMetadata.
47 valid_attn_kwargs = {}
48 for field in dataclasses.fields(attn_backend.get_metadata_cls()):
49 - val = tensor_dict.pop(field.name, None)
50 - if val is not None:
51 - valid_attn_kwargs[field.name] = val
49 + if field.name in tensor_dict:
50 + valid_attn_kwargs[field.name] = tensor_dict.pop(field.name)
51
52 attn_metadata = attn_backend.make_metadata(**valid_attn_kwargs)
53 tensor_dict["attn_metadata"] = attn_metadata
vllm/worker/openvino_model_runner.py
+31 -12
@@ -1,4 +1,5 @@
1 -from typing import List, NamedTuple, Optional, Tuple
1 +from collections import defaultdict
2 +from typing import Dict, List, NamedTuple, Optional, Tuple
3
4 import openvino as ov
5 import torch
@@ -14,7 +15,7 @@ from vllm.model_executor import SamplingMetadata
15 from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.model_executor.model_loader.openvino import get_model
17 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17 - MultiModalInputs)
18 + MultiModalInputs, MultiModalPlaceholderMap)
19 from vllm.sequence import SequenceGroupMetadata
20
21 logger = init_logger(__name__)
@@ -115,6 +116,9 @@ class OpenVINOModelRunner:
116 past_lens: List[int] = []
117 query_lens: List[int] = []
118 multi_modal_inputs_list: List[MultiModalInputs] = []
119 + multi_modal_placeholder_maps: Dict[
120 + str,
121 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
122
123 subsequence_begins: List[int] = []
124 block_indices: List[int] = []
@@ -168,15 +172,6 @@ class OpenVINOModelRunner:
172 and self.sliding_window is None
173 and is_prompt)
174
171 - mm_data = seq_group_metadata.multi_modal_data
172 - if mm_data:
173 - mm_kwargs = self.multi_modal_input_mapper(
174 - mm_data,
175 - mm_processor_kwargs=seq_group_metadata.
176 - mm_processor_kwargs,
177 - )
178 - multi_modal_inputs_list.append(mm_kwargs)
179 -
175 block_table = seq_group_metadata.block_tables[seq_id]
176 # TODO(sang): Combine chunked prefill and prefix caching by
177 # only allowing multiple of block_size chunk size.
@@ -220,7 +215,8 @@ class OpenVINOModelRunner:
215 query_lens.append(query_len)
216
217 input_tokens.extend(tokens)
223 - input_positions.extend(list(range(computed_len, seq_len)))
218 + positions_range = range(computed_len, seq_len)
219 + input_positions.extend(list(positions_range))
220
221 past_lens.append(computed_len)
222 subsequence_begins.append(subsequence_begins[-1] + query_len)
@@ -233,6 +229,22 @@ class OpenVINOModelRunner:
229 ), "seq_len: {}, computed_len: {}, query_len: {}".format(
230 seq_len, computed_len, query_len)
231
232 + if seq_group_metadata.multi_modal_data:
233 + # NOTE: mm_data only includes the subset of multi-modal
234 + # items that intersect with the current prefill positions.
235 + mm_data, placeholder_maps = MultiModalPlaceholderMap \
236 + .from_seq_group(seq_group_metadata, positions_range)
237 +
238 + mm_kwargs = self.multi_modal_input_mapper(
239 + mm_data,
240 + mm_processor_kwargs=seq_group_metadata.
241 + mm_processor_kwargs)
242 + multi_modal_inputs_list.append(mm_kwargs)
243 +
244 + for modality, placeholder_map in placeholder_maps.items():
245 + multi_modal_placeholder_maps[modality].extend(
246 + placeholder_map, )
247 +
248 max_query_len = max(query_lens)
249 assert max_query_len > 0, "query_lens: {}".format(query_lens)
250
@@ -261,12 +273,19 @@ class OpenVINOModelRunner:
273 max_context_len, dtype=torch.int32,
274 device=self.device) # type: ignore
275
276 + placeholder_index_maps = {
277 + modality: placeholder_map.index_map()
278 + for modality, placeholder_map in
279 + multi_modal_placeholder_maps.items()
280 + }
281 +
282 attn_metadata = self.attn_backend.make_openvino_metadata(
283 past_lens=past_lens_tensor,
284 subsequence_begins=subsequence_begins_tensor,
285 block_indices=block_indices_tensor,
286 block_indices_begins=block_indices_begins_tensor,
287 max_context_len=max_context_len_tensor,
288 + multi_modal_placeholder_index_maps=placeholder_index_maps,
289 )
290
291 multi_modal_kwargs = MultiModalInputs.batch(multi_modal_inputs_list)
vllm/worker/tpu_model_runner.py
+4
@@ -184,6 +184,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
184 num_prefill_tokens=batch_size * seq_len,
185 num_decode_tokens=0,
186 slot_mapping=slot_mapping,
187 + multi_modal_placeholder_index_maps=None,
188 block_tables=None,
189 context_lens=None,
190 )
@@ -216,6 +217,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
217 num_prefill_tokens=0,
218 num_decode_tokens=batch_size * seq_len,
219 slot_mapping=slot_mapping,
220 + multi_modal_placeholder_index_maps=None,
221 block_tables=block_tables,
222 context_lens=context_lens,
223 )
@@ -360,6 +362,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
362 num_prefill_tokens=0, # NOTE: This is not used.
363 num_decode_tokens=0,
364 slot_mapping=slot_mapping,
365 + multi_modal_placeholder_index_maps=None,
366 block_tables=None,
367 context_lens=None,
368 )
@@ -429,6 +432,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
432 num_prefill_tokens=0,
433 num_decode_tokens=batch_size,
434 slot_mapping=slot_mapping,
435 + multi_modal_placeholder_index_maps=None,
436 block_tables=block_tables,
437 context_lens=context_lens,
438 )
vllm/worker/xpu_model_runner.py
+32 -6
@@ -1,6 +1,7 @@
1 import dataclasses
2 import time
3 import weakref
4 +from collections import defaultdict
5 from dataclasses import dataclass
6 from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
7 Type, TypeVar)
@@ -19,7 +20,8 @@ from vllm.model_executor import SamplingMetadataCache
20 from vllm.model_executor.layers.sampler import SamplerOutput
21 from vllm.model_executor.model_loader import get_model
22 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
22 - MultiModalInputs, MultiModalRegistry)
23 + MultiModalInputs, MultiModalPlaceholderMap,
24 + MultiModalRegistry)
25 from vllm.sampling_params import SamplingParams
26 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
27 from vllm.utils import DeviceMemoryProfiler, make_tensor_with_pad
@@ -161,6 +163,9 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
163 slot_mapping: List[int] = []
164 seq_lens: List[int] = []
165 multi_modal_inputs_list: List[MultiModalInputs] = []
166 + multi_modal_placeholder_maps: Dict[
167 + str,
168 + MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
169
170 for seq_group_metadata in seq_group_metadata_list:
171 assert seq_group_metadata.is_prompt
@@ -179,7 +184,21 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
184 # Token position ids
185 # NOTE(woosuk): Here we assume that the first token in the prompt
186 # is always the first token in the sequence.
182 - input_positions.extend(list(range(computed_len, seq_len)))
187 + positions_range = range(computed_len, seq_len)
188 + input_positions.extend(list(positions_range))
189 +
190 + if seq_group_metadata.multi_modal_data:
191 + # NOTE: mm_data only includes the subset of multi-modal items
192 + # that intersect with the current prefill positions.
193 + mm_data, placeholder_maps = MultiModalPlaceholderMap \
194 + .from_seq_group(seq_group_metadata, positions_range)
195 +
196 + mm_kwargs = self.runner.multi_modal_input_mapper(mm_data)
197 + multi_modal_inputs_list.append(mm_kwargs)
198 +
199 + for modality, placeholder_map in placeholder_maps.items():
200 + multi_modal_placeholder_maps[modality].extend(
201 + placeholder_map)
202
203 if seq_group_metadata.block_tables is None:
204 # During memory profiling, the block tables are not initialized
@@ -220,6 +239,11 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
239 slot_mapping = torch.tensor(slot_mapping,
240 dtype=torch.long,
241 device=self.device) # type: ignore
242 + placeholder_index_maps = {
243 + modality: placeholder_map.index_map()
244 + for modality, placeholder_map in
245 + multi_modal_placeholder_maps.items()
246 + }
247
248 max_seqlen = max(seq_lens)
249 tmp = [0]
@@ -230,6 +254,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
254 attn_metadata = self.attn_backend.make_metadata(
255 is_prompt=True,
256 slot_mapping=slot_mapping,
257 + multi_modal_placeholder_index_maps=placeholder_index_maps,
258 seq_lens=seq_lens,
259 seqlen_q=seqlen_q,
260 max_seqlen=max_seqlen,
@@ -313,6 +338,7 @@ class ModelInputForXPUBuilder(ModelRunnerInputBuilderBase[ModelInputForXPU]):
338 attn_metadata = self.attn_backend.make_metadata(
339 is_prompt=False,
340 slot_mapping=slot_mapping,
341 + multi_modal_placeholder_index_maps=None,
342 seq_lens=seq_lens,
343 seqlen_q=torch.tensor([]),
344 max_seqlen=0,
@@ -450,7 +476,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
476 (group_id < max_num_batched_tokens % max_num_seqs))
477 batch_size += seq_len
478
453 - seq_data, dummy_multi_modal_data = self.input_registry \
479 + dummy_data = self.input_registry \
480 .dummy_data_for_profiling(self.model_config,
481 seq_len,
482 self.mm_registry)
@@ -458,12 +484,12 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
484 seq = SequenceGroupMetadata(
485 request_id=str(group_id),
486 is_prompt=True,
461 - seq_data={group_id: seq_data},
487 + seq_data={group_id: dummy_data.seq_data},
488 sampling_params=sampling_params,
489 block_tables=None,
490 lora_request=None,
465 - multi_modal_data=dummy_multi_modal_data,
466 - )
491 + multi_modal_data=dummy_data.multi_modal_data,
492 + multi_modal_placeholders=dummy_data.multi_modal_placeholders)
493 seqs.append(seq)
494
495 # Run the model with the dummy inputs.