A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[V1] Scatter and gather placeholders in the model runner (#15712)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Apr 5, 2025 at 05:26 UTC
f5722a505222c41d21571d4506d7f8cd78020f7e
42 files changed
+497
-943
docs/source/contributing/model/multimodal.md
+8
-8
@@ -860,8 +860,8 @@ prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch(
860
)
861
```
862
863
-To accommodate this, instead of a string you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`
864
-with different `full` and `feature` attributes:
863
+To assign the vision embeddings to only the image tokens, instead of a string
864
+you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`:
865
866
```python
867
hf_config = self.info.get_hf_config()
@@ -879,9 +879,9 @@ def get_replacement_fuyu(item_idx: int):
879
image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
880
[_NEWLINE_TOKEN_ID]) * nrows
881
882
- return PromptUpdateDetails(
883
- full=image_tokens + [bos_token_id],
884
- features=image_tokens,
882
+ return PromptUpdateDetails.select_token_id(
883
+ image_tokens + [bos_token_id],
884
+ embed_token_id=_IMAGE_TOKEN_ID,
885
)
886
```
887
@@ -914,9 +914,9 @@ def _get_prompt_updates(
914
image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
915
[_NEWLINE_TOKEN_ID]) * nrows
916
917
- return PromptUpdateDetails(
918
- full=image_tokens + [bos_token_id],
919
- features=image_tokens,
917
+ return PromptUpdateDetails.select_token_id(
918
+ image_tokens + [bos_token_id],
919
+ embed_token_id=_IMAGE_TOKEN_ID,
920
)
921
922
return [
docs/source/models/supported_models.md
-3
@@ -989,9 +989,6 @@ See [this page](#generative-models) for more information on how to use generativ
989
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
990
991
:::{important}
992
-To use Gemma3 series models, you have to install Hugging Face Transformers library from source via
993
-`pip install git+https://github.com/huggingface/transformers`.
994
-
992
Pan-and-scan image pre-processing is currently supported on V0 (but not V1).
993
You can enable it by passing `--mm-processor-kwargs '{"do_pan_and_scan": True}'`.
994
:::
examples/offline_inference/audio_language.py
+1
-1
@@ -47,7 +47,7 @@ def run_minicpmo(question: str, audio_count: int) -> ModelRequestData:
47
model=model_name,
48
trust_remote_code=True,
49
max_model_len=4096,
50
- max_num_seqs=5,
50
+ max_num_seqs=2,
51
limit_mm_per_prompt={"audio": audio_count},
52
)
53
tests/models/decoder_only/audio_language/test_ultravox.py
+4
-1
@@ -55,7 +55,10 @@ def server(request, audio_assets):
55
for key, value in request.param.items()
56
]
57
58
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
58
+ with RemoteOpenAIServer(MODEL_NAME,
59
+ args,
60
+ env_dict={"VLLM_AUDIO_FETCH_TIMEOUT":
61
+ "30"}) as remote_server:
62
yield remote_server
63
64
tests/models/decoder_only/vision_language/test_models.py
+1
-1
@@ -167,7 +167,7 @@ VLM_TEST_SETTINGS = {
167
"cherry_blossom": "<image>What is the season?", # noqa: E501
168
}),
169
multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
170
- max_model_len=8192,
170
+ max_model_len=4096,
171
max_num_seqs=2,
172
auto_cls=AutoModelForImageTextToText,
173
vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}}
tests/models/decoder_only/vision_language/test_pixtral.py
+10
-16
@@ -176,6 +176,8 @@ def test_chat(
176
model,
177
dtype=dtype,
178
tokenizer_mode="mistral",
179
+ load_format="mistral",
180
+ config_format="mistral",
181
max_model_len=max_model_len,
182
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
183
) as vllm_model:
@@ -198,22 +200,14 @@ def test_chat(
200
201
202
@large_gpu_test(min_gb=48)
201
-@pytest.mark.parametrize(
202
- "prompt,expected_ranges",
203
- [(_create_engine_inputs_hf(IMG_URLS[:1]), [{
204
- "offset": 11,
205
- "length": 494
206
- }]),
207
- (_create_engine_inputs_hf(IMG_URLS[1:4]), [{
208
- "offset": 11,
209
- "length": 266
210
- }, {
211
- "offset": 277,
212
- "length": 1056
213
- }, {
214
- "offset": 1333,
215
- "length": 418
216
- }])])
203
+@pytest.mark.parametrize("prompt,expected_ranges",
204
+ [(_create_engine_inputs_hf(IMG_URLS[:1]),
205
+ [PlaceholderRange(offset=11, length=494)]),
206
+ (_create_engine_inputs_hf(IMG_URLS[1:4]), [
207
+ PlaceholderRange(offset=11, length=266),
208
+ PlaceholderRange(offset=277, length=1056),
209
+ PlaceholderRange(offset=1333, length=418)
210
+ ])])
211
def test_multi_modal_placeholders(vllm_runner, prompt,
212
expected_ranges: list[PlaceholderRange],
213
monkeypatch) -> None:
tests/models/multimodal/processing/test_llava_next.py
+2
-2
@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(
92
first_placeholder = image_placeholders[0]
93
94
# NOTE: There is a BOS token
95
- assert first_placeholder["offset"] == 1
96
- assert first_placeholder["length"] == (
95
+ assert first_placeholder.offset == 1
96
+ assert first_placeholder.length == (
97
len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
98
99
except Exception as exc:
tests/models/multimodal/processing/test_llava_onevision.py
+2
-2
@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(
92
93
first_placeholder = image_placeholders[0]
94
95
- assert first_placeholder["offset"] == 0
96
- assert first_placeholder["length"] == len(
95
+ assert first_placeholder.offset == 0
96
+ assert first_placeholder.length == len(
97
processed_inputs["prompt_token_ids"]) // num_imgs
98
except Exception as exc:
99
failed_size_excs.append((image_size, exc))
tests/models/registry.py
+3
-1
@@ -277,7 +277,9 @@ _MULTIMODAL_EXAMPLE_MODELS = {
277
trust_remote_code=True,
278
hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
279
"H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
280
- extras={"2b": "h2oai/h2ovl-mississippi-2b"}), # noqa: E501
280
+ extras={"2b": "h2oai/h2ovl-mississippi-2b"}, # noqa: E501
281
+ max_transformers_version="4.48", # noqa: E501
282
+ transformers_version_reason="HF model is not compatible."), # noqa: E501
283
"InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
284
extras={"2B": "OpenGVLab/InternVL2-2B"}, # noqa: E501
285
trust_remote_code=True),
tests/multimodal/test_processing.py
+9
@@ -785,6 +785,7 @@ def test_find_update_tokens(
785
item_idx=0,
786
start_idx=6,
787
tokens=[32000, 32000],
788
+ is_embed=None,
789
),
790
],
791
"pattern_4": [
@@ -793,6 +794,7 @@ def test_find_update_tokens(
794
item_idx=0,
795
start_idx=3,
796
tokens=[32000],
797
+ is_embed=None,
798
),
799
],
800
}
@@ -807,12 +809,14 @@ def test_find_update_tokens(
809
item_idx=0,
810
start_idx=1,
811
tokens=[32000, 32000],
812
+ is_embed=None,
813
),
814
PlaceholderFeaturesInfo(
815
modality="pattern_1",
816
item_idx=1,
817
start_idx=5,
818
tokens=[32000, 32000],
819
+ is_embed=None,
820
),
821
],
822
"pattern_3": [
@@ -821,6 +825,7 @@ def test_find_update_tokens(
825
item_idx=0,
826
start_idx=7,
827
tokens=[1550, 918, 1550],
828
+ is_embed=None,
829
),
830
],
831
# No match for pattern_4 as it has lower priority than pattern_1
@@ -835,12 +840,14 @@ def test_find_update_tokens(
840
item_idx=0,
841
start_idx=1,
842
tokens=[32000, 32000],
843
+ is_embed=None,
844
),
845
PlaceholderFeaturesInfo(
846
modality="pattern_1",
847
item_idx=1,
848
start_idx=3,
849
tokens=[32000, 32000],
850
+ is_embed=None,
851
),
852
],
853
"pattern_4": [
@@ -849,6 +856,7 @@ def test_find_update_tokens(
856
item_idx=0,
857
start_idx=5,
858
tokens=[32000],
859
+ is_embed=None,
860
),
861
],
862
"pattern_3": [
@@ -857,6 +865,7 @@ def test_find_update_tokens(
865
item_idx=0,
866
start_idx=6,
867
tokens=[1550, 918, 1550],
868
+ is_embed=None,
869
),
870
],
871
}
tests/v1/core/test_kv_cache_utils.py
+17
-29
@@ -3,7 +3,7 @@
3
import pytest
4
import torch
5
6
-from vllm.multimodal.inputs import MultiModalKwargs
6
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
7
from vllm.sampling_params import SamplingParams
8
from vllm.utils import sha256
9
# disable yapf here as it formats differently than isort such that both fail
@@ -158,13 +158,10 @@ def test_generate_block_hash_extra_keys():
158
request = make_request(
159
request_id=0,
160
prompt_token_ids=[_ for _ in range(20)],
161
- mm_positions=[{
162
- "offset": 0,
163
- "length": 5
164
- }, {
165
- "offset": 10,
166
- "length": 5
167
- }],
161
+ mm_positions=[
162
+ PlaceholderRange(offset=0, length=5),
163
+ PlaceholderRange(offset=10, length=5),
164
+ ],
165
mm_hashes=["hash1", "hash2"],
166
)
167
@@ -222,13 +219,10 @@ def test_hash_request_tokens(hash_fn):
219
request = make_request(
220
request_id=0,
221
prompt_token_ids=[_ for _ in range(6)],
225
- mm_positions=[{
226
- "offset": 0,
227
- "length": 3
228
- }, {
229
- "offset": 3,
230
- "length": 3
231
- }],
222
+ mm_positions=[
223
+ PlaceholderRange(offset=0, length=3),
224
+ PlaceholderRange(offset=3, length=3),
225
+ ],
226
mm_hashes=["hash1", "hash2"],
227
)
228
@@ -253,25 +247,19 @@ def test_hash_tokens_different_mm_input(hash_fn):
247
request1 = make_request(
248
request_id=0,
249
prompt_token_ids=[_ for _ in range(6)],
256
- mm_positions=[{
257
- "offset": 0,
258
- "length": 3
259
- }, {
260
- "offset": 3,
261
- "length": 3
262
- }],
250
+ mm_positions=[
251
+ PlaceholderRange(offset=0, length=3),
252
+ PlaceholderRange(offset=3, length=3),
253
+ ],
254
mm_hashes=["hash1", "hash2"],
255
)
256
request2 = make_request(
257
request_id=1,
258
prompt_token_ids=[_ for _ in range(6)],
268
- mm_positions=[{
269
- "offset": 0,
270
- "length": 3
271
- }, {
272
- "offset": 3,
273
- "length": 3
274
- }],
259
+ mm_positions=[
260
+ PlaceholderRange(offset=0, length=3),
261
+ PlaceholderRange(offset=3, length=3),
262
+ ],
263
mm_hashes=["hash3", "hash2"],
264
)
265
block_size = 3
vllm/model_executor/models/aya_vision.py
+21
-50
@@ -27,7 +27,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
27
BaseProcessingInfo,
28
MultiModalFieldConfig,
29
PromptReplacement, PromptUpdate,
30
- encode_tokens)
30
+ PromptUpdateDetails)
31
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32
from vllm.sequence import IntermediateTensors
33
@@ -35,7 +35,6 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35
from .siglip import SiglipVisionModel
36
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
37
maybe_prefix, merge_multimodal_embeddings)
38
-from .vision import scatter_patch_features, select_patch_features
38
39
40
class AyaVisionImagePixelInputs(TypedDict):
@@ -51,13 +50,6 @@ class AyaVisionImagePixelInputs(TypedDict):
50
num_patches: torch.Tensor
51
"""Shape: `(batch_size * num_images)`"""
52
54
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
55
- """
56
- A boolean mask indicating which image embeddings correspond to patch tokens.
57
-
58
- Shape: `(batch_size * num_images, num_embeds)`
59
- """
60
-
53
54
class AyaVisionMultiModalProjector(nn.Module):
55
@@ -135,21 +127,20 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
127
def get_max_image_tokens(self) -> int:
128
hf_processor = self.get_hf_processor()
129
image_processor = hf_processor.image_processor
130
+
131
image_size = self.get_image_size_with_most_features()
139
- tokenizer = hf_processor.tokenizer
132
num_patches = self.get_num_patches(
133
image_width=image_size.width,
134
image_height=image_size.height,
135
size=image_processor.size,
136
min_patches=image_processor.min_patches,
145
- max_patches=image_processor.max_patches)
146
- image_string = hf_processor._prompt_split_image(num_patches)
147
- x = encode_tokens(
148
- tokenizer,
149
- image_string,
150
- add_special_tokens=False,
137
+ max_patches=image_processor.max_patches,
138
)
152
- return len(x)
139
+
140
+ img_patches_per_tile = (hf_processor.img_size //
141
+ hf_processor.patch_size)**2
142
+
143
+ return num_patches * img_patches_per_tile
144
145
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
146
return {"image": None}
@@ -221,7 +212,6 @@ class AyaVisionMultiModalProcessor(
212
hf_processor = self.info.get_hf_processor(**mm_kwargs)
213
image_processor = hf_processor.image_processor
214
224
- hf_config = self.info.get_hf_config()
215
# HF processor pops the `num_patches` kwarg, which is needed by vLLM
216
if (images :=
217
mm_data.get("images")) is not None and '<image>' in prompt:
@@ -234,6 +224,7 @@ class AyaVisionMultiModalProcessor(
224
parsed_images.get_image_size(i)
225
for i in range(len(parsed_images))
226
]
227
+
228
num_patches = [
229
self.info.get_num_patches(
230
image_width=image_size.width,
@@ -243,20 +234,6 @@ class AyaVisionMultiModalProcessor(
234
max_patches=image_processor.max_patches)
235
for image_size in image_sizes
236
]
246
- image_tokens_list = [
247
- hf_processor._prompt_split_image(num_patch)
248
- for num_patch in num_patches
249
- ]
250
- tokenizer = self.info.get_tokenizer()
251
- image_token_ids = [
252
- tokenizer.encode(image_tokens, add_special_tokens=False)
253
- for image_tokens in image_tokens_list
254
- ]
255
- embed_is_patch = [
256
- torch.tensor(image_repl_tokens) == hf_config.image_token_index
257
- for image_repl_tokens in image_token_ids
258
- ]
259
- processed_outputs["embed_is_patch"] = embed_is_patch
237
processed_outputs["num_patches"] = torch.tensor(num_patches)
238
239
return processed_outputs
@@ -271,7 +248,6 @@ class AyaVisionMultiModalProcessor(
248
pixel_values=MultiModalFieldConfig.flat_from_sizes(
249
"image", num_patches),
250
num_patches=MultiModalFieldConfig.batched("image"),
274
- embed_is_patch=MultiModalFieldConfig.batched("image"),
251
image_embeds=MultiModalFieldConfig.batched("image"),
252
)
253
@@ -283,6 +259,7 @@ class AyaVisionMultiModalProcessor(
259
) -> Sequence[PromptUpdate]:
260
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
261
image_token = hf_processor.image_token
262
+ img_patch_token = hf_processor.img_patch_token
263
image_processor = hf_processor.image_processor
264
265
def get_replacement(item_idx: int):
@@ -294,8 +271,11 @@ class AyaVisionMultiModalProcessor(
271
image_height=image_size.height,
272
size=image_processor.size,
273
min_patches=image_processor.min_patches,
297
- max_patches=image_processor.max_patches)
298
- return hf_processor._prompt_split_image(num_patches=num_patches)
274
+ max_patches=image_processor.max_patches,
275
+ )
276
+ repl = hf_processor._prompt_split_image(num_patches=num_patches)
277
+
278
+ return PromptUpdateDetails.select_text(repl, img_patch_token)
279
280
return [
281
PromptReplacement(
@@ -424,7 +404,6 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
404
self, **kwargs: object) -> Optional[AyaVisionImagePixelInputs]:
405
pixel_values = kwargs.pop("pixel_values", None)
406
num_patches = kwargs.pop("num_patches", None)
427
- embed_is_patch = kwargs.pop("embed_is_patch", None)
407
image_embeds = kwargs.pop("image_embeds", None)
408
assert image_embeds is None, "Aya Vision does not support image_embeds."
409
@@ -436,18 +415,13 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
415
raise ValueError("Incorrect type of num_patches. "
416
f"Got type: {type(num_patches)}")
417
439
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
440
- raise ValueError("Incorrect type of embed_is_patch. "
441
- f"Got type: {type(embed_is_patch)}")
442
-
418
pixel_values = flatten_bn(pixel_values, concat=True)
419
num_patches = flatten_bn(num_patches, concat=True)
445
- embed_is_patch = flatten_bn(embed_is_patch)
420
+
421
return AyaVisionImagePixelInputs(
422
type="pixel_values",
423
pixel_values=self._validate_pixel_values(pixel_values),
424
num_patches=num_patches,
450
- embed_is_patch=embed_is_patch,
425
)
426
427
def get_multimodal_embeddings(
@@ -455,11 +429,8 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
429
image_input = self._parse_and_validate_image_input(**kwargs)
430
if image_input is None:
431
return None
458
- image_features = self._process_image_input(image_input, **kwargs)
459
- return scatter_patch_features(
460
- image_features,
461
- image_input["embed_is_patch"],
462
- )
432
+
433
+ return self._process_image_input(image_input, **kwargs)
434
435
def get_input_embeddings(
436
self,
@@ -471,9 +442,9 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
442
inputs_embeds = merge_multimodal_embeddings(
443
input_ids=input_ids,
444
inputs_embeds=inputs_embeds,
474
- multimodal_embeddings=select_patch_features(
475
- multimodal_embeddings),
476
- placeholder_token_id=self.config.image_token_index)
445
+ multimodal_embeddings=multimodal_embeddings,
446
+ placeholder_token_id=self.config.image_token_index,
447
+ )
448
449
return inputs_embeds
450
vllm/model_executor/models/chameleon.py
+3
-3
@@ -162,9 +162,9 @@ class ChameleonMultiModalProcessor(
162
PromptReplacement(
163
modality="image",
164
target=[image_token_id],
165
- replacement=PromptUpdateDetails(
166
- full=([image_start_id] + image_tokens + [image_end_id]),
167
- features=image_tokens,
165
+ replacement=PromptUpdateDetails.select_token_id(
166
+ [image_start_id] + image_tokens + [image_end_id],
167
+ embed_token_id=image_token_id,
168
),
169
)
170
]
vllm/model_executor/models/fuyu.py
+33
-52
@@ -18,7 +18,7 @@
18
""" PyTorch Fuyu model."""
19
import math
20
from collections.abc import Iterable, Mapping, Sequence
21
-from typing import Literal, Optional, Set, Tuple, TypedDict, Union
21
+from typing import Literal, Optional, Set, Tuple, TypedDict
22
23
import torch
24
import torch.nn as nn
@@ -43,7 +43,6 @@ from vllm.sequence import IntermediateTensors
43
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
44
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
45
merge_multimodal_embeddings)
46
-from .vision import scatter_patch_features, select_patch_features
46
47
# Cannot find the following 2 numbers from hf config.
48
_IMAGE_TOKEN_ID = 71011
@@ -66,14 +65,6 @@ class FuyuImagePatchInputs(TypedDict):
65
flattened just like `flat_data`.
66
"""
67
69
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
70
- """
71
- A boolean mask indicating which image embeddings correspond
72
- to patch tokens.
73
-
74
- Shape: `(batch_size * num_images, num_embeds)`
75
- """
76
-
68
69
class FuyuProcessingInfo(BaseProcessingInfo):
70
@@ -94,15 +85,7 @@ class FuyuProcessingInfo(BaseProcessingInfo):
85
seq_len: int,
86
mm_counts: Mapping[str, int],
87
) -> Mapping[str, int]:
97
- target_width, target_height = self.get_image_size_with_most_features()
98
-
99
- max_ncols, max_nrows = self.get_image_feature_grid_size(
100
- image_width=target_width,
101
- image_height=target_height,
102
- )
103
- max_image_tokens = (max_ncols + 1) * max_nrows
104
-
105
- return {"image": max_image_tokens}
88
+ return {"image": self.get_max_image_tokens()}
89
90
def get_image_feature_grid_size(
91
self,
@@ -128,11 +111,32 @@ class FuyuProcessingInfo(BaseProcessingInfo):
111
nrows = math.ceil(image_height / patch_height)
112
return ncols, nrows
113
114
+ def get_num_image_tokens(
115
+ self,
116
+ *,
117
+ image_width: int,
118
+ image_height: int,
119
+ ) -> int:
120
+ ncols, nrows = self.get_image_feature_grid_size(
121
+ image_width=image_width,
122
+ image_height=image_height,
123
+ )
124
+
125
+ return ncols * nrows
126
+
127
def get_image_size_with_most_features(self) -> ImageSize:
128
image_processor = self.get_image_processor()
129
return ImageSize(width=image_processor.size["width"],
130
height=image_processor.size["height"])
131
132
+ def get_max_image_tokens(self) -> int:
133
+ target_width, target_height = self.get_image_size_with_most_features()
134
+
135
+ return self.get_num_image_tokens(
136
+ image_width=target_width,
137
+ image_height=target_height,
138
+ )
139
+
140
141
class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
142
@@ -192,19 +196,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
196
197
processed_outputs["image_patches"] = image_patches[0]
198
195
- # get patch grid size for each image
196
- embed_is_patch = []
197
- for image in images:
198
- ncols, nrows = self.info.get_image_feature_grid_size(
199
- image_width=image.width,
200
- image_height=image.height,
201
- )
202
-
203
- mask = torch.tensor(([True] * ncols + [False]) * nrows)
204
- embed_is_patch.append(mask)
205
-
206
- processed_outputs["embed_is_patch"] = embed_is_patch
207
-
199
return processed_outputs
200
201
def _apply_hf_processor_tokens_only(
@@ -224,8 +215,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
215
hf_inputs: BatchFeature,
216
hf_processor_mm_kwargs: Mapping[str, object],
217
) -> Mapping[str, MultiModalFieldConfig]:
227
- return dict(image_patches=MultiModalFieldConfig.batched("image"),
228
- embed_is_patch=MultiModalFieldConfig.batched("image"))
218
+ return dict(image_patches=MultiModalFieldConfig.batched("image"))
219
220
def _get_prompt_updates(
221
self,
@@ -252,9 +242,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
242
image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
243
[_NEWLINE_TOKEN_ID]) * nrows
244
255
- return PromptUpdateDetails(
256
- full=image_tokens + [bos_token_id],
257
- features=image_tokens,
245
+ return PromptUpdateDetails.select_token_id(
246
+ image_tokens + [bos_token_id],
247
+ embed_token_id=_IMAGE_TOKEN_ID,
248
)
249
250
return [
@@ -329,20 +319,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
319
raise ValueError("Incorrect type of image patches. "
320
f"Got type: {type(image_patches)}")
321
332
- embed_is_patch = kwargs.pop("embed_is_patch")
333
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
334
- raise ValueError("Incorrect type of embed_is_patch. "
335
- f"Got type: {type(embed_is_patch)}")
336
-
322
image_patches_flat = flatten_bn(image_patches)
338
- embed_is_patch = flatten_bn(embed_is_patch)
323
324
return FuyuImagePatchInputs(
325
type="image_patches",
326
flat_data=self._validate_pixel_values(
327
flatten_bn(image_patches_flat, concat=True)),
328
patches_per_image=[x.size(0) for x in image_patches_flat],
345
- embed_is_patch=embed_is_patch,
329
)
330
331
return None
@@ -364,12 +347,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
347
if image_input is None:
348
return None
349
367
- image_features = self._process_image_input(image_input)
368
-
369
- return scatter_patch_features(
370
- image_features,
371
- image_input["embed_is_patch"],
372
- )
350
+ return self._process_image_input(image_input)
351
352
def get_input_embeddings(
353
self,
@@ -379,8 +357,11 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
357
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
358
if multimodal_embeddings is not None:
359
inputs_embeds = merge_multimodal_embeddings(
382
- input_ids, inputs_embeds,
383
- select_patch_features(multimodal_embeddings), _IMAGE_TOKEN_ID)
360
+ input_ids,
361
+ inputs_embeds,
362
+ multimodal_embeddings,
363
+ _IMAGE_TOKEN_ID,
364
+ )
365
return inputs_embeds
366
367
def forward(
vllm/model_executor/models/gemma3_mm.py
+20
-63
@@ -25,7 +25,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
25
PlaceholderFeaturesInfo,
26
PromptReplacement, PromptTargetMatch,
27
PromptUpdate, PromptUpdateDetails,
28
- encode_tokens, find_mm_placeholders,
28
+ find_mm_placeholders,
29
replace_token_matches)
30
# yapf: enable
31
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -36,7 +36,6 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
36
from .siglip import SiglipVisionModel
37
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
maybe_prefix, merge_multimodal_embeddings)
39
-from .vision import scatter_patch_features, select_patch_features
39
40
logger = init_logger(__name__)
41
@@ -54,14 +53,6 @@ class Gemma3ImagePixelInputs(TypedDict):
53
num_patches: torch.Tensor
54
"""Shape: `(batch_size * num_images)`"""
55
57
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
58
- """
59
- A boolean mask indicating which image embeddings correspond
60
- to patch tokens.
61
-
62
- Shape: `(batch_size * num_images, num_embeds)`
63
- """
64
-
56
57
Gemma3ImageInputs = Gemma3ImagePixelInputs
58
@@ -183,7 +174,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
174
if processor is None:
175
processor = self.get_hf_processor()
176
186
- image_token = processor.boi_token
177
+ boi_token = processor.boi_token
178
179
num_crops = self.get_num_crops(
180
image_width=image_width,
@@ -192,19 +183,21 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
183
)
184
185
if num_crops == 0:
195
- image_text = image_token
186
+ image_text = boi_token
187
else:
197
- crops_image_tokens = " ".join(image_token
198
- for _ in range(num_crops))
188
+ crops_image_tokens = " ".join(boi_token for _ in range(num_crops))
189
image_text = (
200
- f"Here is the original image {image_token} and here are some "
190
+ f"Here is the original image {boi_token} and here are some "
191
f"crops to help you see better {crops_image_tokens}")
192
203
- repl_full = image_text.replace(image_token,
193
+ repl_full = image_text.replace(boi_token,
194
processor.full_image_sequence)
205
- repl_features = repl_full.strip("\n")
195
207
- return PromptUpdateDetails(full=repl_full, features=repl_features)
196
+ tokenizer = processor.tokenizer
197
+ vocab = tokenizer.get_vocab()
198
+ image_token_id = vocab[tokenizer.image_token]
199
+
200
+ return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
201
202
def get_num_image_tokens(
203
self,
@@ -213,19 +206,17 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
206
image_height: int,
207
processor: Optional[Gemma3Processor],
208
) -> int:
216
- tokenizer = self.get_tokenizer()
217
- image_repl = self.get_image_repl(
209
+ if processor is None:
210
+ processor = self.get_hf_processor()
211
+
212
+ num_crops = self.get_num_crops(
213
image_width=image_width,
214
image_height=image_height,
215
processor=processor,
216
)
217
+ image_seq_len = processor.image_seq_length
218
223
- image_repl_tokens = encode_tokens(
224
- tokenizer,
225
- image_repl.features,
226
- add_special_tokens=False,
227
- )
228
- return len(image_repl_tokens)
219
+ return (num_crops + 1) * image_seq_len
220
221
def get_image_size_with_most_features(self) -> ImageSize:
222
processor = self.get_hf_processor()
@@ -301,28 +292,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
292
]
293
hf_processor = self.info.get_hf_processor(**mm_kwargs)
294
304
- image_repl_features = [
305
- self.info.get_image_repl(image_width=size.width,
306
- image_height=size.height,
307
- processor=hf_processor).features
308
- for size in image_sizes
309
- ]
310
-
311
- tokenizer = self.info.get_tokenizer()
312
- image_repls_feature_tokens = [
313
- tokenizer.encode(image_repl, add_special_tokens=False)
314
- for image_repl in image_repl_features
315
- ]
316
-
317
- vocab = tokenizer.get_vocab()
318
- image_token_id = vocab[tokenizer.image_token]
319
-
320
- embed_is_patch = [
321
- torch.tensor(image_repl_tokens) == image_token_id
322
- for image_repl_tokens in image_repls_feature_tokens
323
- ]
324
- processed_outputs["embed_is_patch"] = embed_is_patch
325
-
295
num_crops = [
296
self.info.get_num_crops(image_width=size.width,
297
image_height=size.height,
@@ -344,7 +313,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
313
pixel_values=MultiModalFieldConfig.flat_from_sizes(
314
"image", num_crops + 1),
315
num_crops=MultiModalFieldConfig.batched("image"),
347
- embed_is_patch=MultiModalFieldConfig.batched("image"),
316
)
317
318
def _get_prompt_updates(
@@ -454,6 +422,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
422
item_idx=p.item_idx,
423
start_idx=repl_orig_idxs[p.start_idx],
424
tokens=p.tokens,
425
+ is_embed=p.is_embed,
426
) for p in placeholders
427
]
428
for modality, placeholders in repls.items()
@@ -572,7 +541,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
541
self, **kwargs: object) -> Optional[Gemma3ImageInputs]:
542
pixel_values = kwargs.pop("pixel_values", None)
543
num_crops = kwargs.pop("num_crops", None)
575
- embed_is_patch = kwargs.pop("embed_is_patch", None)
544
image_embeds = kwargs.pop("image_embeds", None)
545
assert image_embeds is None, "Gemma3 does not support image_embeds."
546
if pixel_values is None:
@@ -586,19 +554,13 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
554
raise ValueError("Incorrect type of num_crops. "
555
f"Got type: {type(num_crops)}")
556
589
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
590
- raise ValueError("Incorrect type of embed_is_patch. "
591
- f"Got type: {type(embed_is_patch)}")
592
-
557
pixel_values = flatten_bn(pixel_values, concat=True)
558
num_crops = flatten_bn(num_crops, concat=True)
595
- embed_is_patch = flatten_bn(embed_is_patch)
559
560
return Gemma3ImagePixelInputs(
561
type="pixel_values",
562
pixel_values=self._validate_pixel_values(pixel_values),
563
num_patches=num_crops + 1,
601
- embed_is_patch=embed_is_patch,
564
)
565
566
def _image_pixels_to_features(
@@ -635,12 +597,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
597
if image_input is None:
598
return None
599
638
- image_features = self._process_image_input(image_input)
639
-
640
- return scatter_patch_features(
641
- image_features,
642
- image_input["embed_is_patch"],
643
- )
600
+ return self._process_image_input(image_input)
601
602
def get_input_embeddings(
603
self,
@@ -652,7 +609,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
609
inputs_embeds = merge_multimodal_embeddings(
610
input_ids,
611
inputs_embeds,
655
- select_patch_features(multimodal_embeddings),
612
+ multimodal_embeddings,
613
self.config.image_token_index,
614
)
615
return inputs_embeds
vllm/model_executor/models/h2ovl.py
+1
-1
@@ -257,7 +257,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
257
repl_features = IMG_CONTEXT * feature_size
258
repl_full = IMG_START + repl_features + IMG_END
259
260
- return PromptUpdateDetails(full=repl_full, features=repl_features)
260
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
261
262
def resolve_min_max_num(
263
self,
vllm/model_executor/models/idefics3.py
+15
-67
@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
MultiModalDataItems,
42
MultiModalFieldConfig,
43
PromptReplacement, PromptUpdate,
44
- encode_tokens)
44
+ PromptUpdateDetails)
45
# yapf: enable
46
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47
from vllm.sequence import IntermediateTensors
@@ -54,7 +54,6 @@ from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
54
from .llama import LlamaModel
55
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
56
merge_multimodal_embeddings)
57
-from .vision import scatter_patch_features, select_patch_features
57
58
59
class Idefics3ImagePixelInputs(TypedDict):
@@ -69,14 +68,6 @@ class Idefics3ImagePixelInputs(TypedDict):
68
num_patches: torch.Tensor
69
"""Shape: `(batch_size * num_images)`"""
70
72
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
73
- """
74
- A boolean mask indicating which image embeddings correspond
75
- to patch tokens.
76
-
77
- Shape: `(batch_size * num_images, num_embeds)`
78
- """
79
-
71
72
class Idefics3ImageEmbeddingInputs(TypedDict):
73
type: Literal["image_embeds"]
@@ -86,14 +77,6 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
77
`hidden_size` must match the hidden size of language model backbone.
78
"""
79
89
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
90
- """
91
- A boolean mask indicating which image embeddings correspond
92
- to patch tokens.
93
-
94
- Shape: `(batch_size * num_images, num_embeds)`
95
- """
96
-
80
81
ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
82
@@ -275,19 +258,16 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
258
image_height: int,
259
processor: Optional[Idefics3Processor],
260
) -> int:
278
- tokenizer = self.get_tokenizer()
279
- image_repl = self.get_image_repl(
261
+ if processor is None:
262
+ processor = self.get_hf_processor()
263
+
264
+ num_patches = self.get_num_patches(
265
image_width=image_width,
266
image_height=image_height,
267
processor=processor,
268
)
269
285
- image_repl_tokens = encode_tokens(
286
- tokenizer,
287
- image_repl,
288
- add_special_tokens=False,
289
- )
290
- return len(image_repl_tokens)
270
+ return num_patches * processor.image_seq_len
271
272
def get_image_size_with_most_features(self) -> ImageSize:
273
processor = self.get_hf_processor()
@@ -364,28 +344,6 @@ class Idefics3MultiModalProcessor(
344
]
345
hf_processor = self.info.get_hf_processor(**mm_kwargs)
346
367
- image_repl_features = [
368
- self.info.get_image_repl(image_width=size.width,
369
- image_height=size.height,
370
- processor=hf_processor)
371
- for size in image_sizes
372
- ]
373
-
374
- tokenizer = self.info.get_tokenizer()
375
- image_repls_feature_tokens = [
376
- tokenizer.encode(image_repl, add_special_tokens=False)
377
- for image_repl in image_repl_features
378
- ]
379
-
380
- vocab = tokenizer.get_vocab()
381
- image_token_id = vocab[hf_processor.image_token.content]
382
-
383
- embed_is_patch = [
384
- torch.tensor(image_repl_tokens) == image_token_id
385
- for image_repl_tokens in image_repls_feature_tokens
386
- ]
387
- processed_outputs["embed_is_patch"] = embed_is_patch
388
-
347
num_patches = [
348
self.info.get_num_patches(
349
image_width=size.width,
@@ -415,7 +373,6 @@ class Idefics3MultiModalProcessor(
373
"image", num_patches),
374
image_embeds=MultiModalFieldConfig.batched("image"),
375
num_patches=MultiModalFieldConfig.batched("image"),
418
- embed_is_patch=MultiModalFieldConfig.batched("image"),
376
)
377
378
def _get_prompt_updates(
@@ -427,17 +384,22 @@ class Idefics3MultiModalProcessor(
384
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
385
image_token = hf_processor.image_token.content
386
430
- def get_replacement_idefics3(item_idx: int) -> str:
387
+ def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
388
images = mm_items.get_items("image", ImageProcessorItems)
389
390
image_size = images.get_image_size(item_idx)
391
435
- return self.info.get_image_repl(
392
+ image_repl = self.info.get_image_repl(
393
image_width=image_size.width,
394
image_height=image_size.height,
395
processor=hf_processor,
396
)
397
398
+ return PromptUpdateDetails.select_text(
399
+ image_repl,
400
+ embed_text=image_token,
401
+ )
402
+
403
return [
404
PromptReplacement(
405
modality="image",
@@ -675,13 +637,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
637
if pixel_values is None and image_embeds is None:
638
return None
639
678
- embed_is_patch = kwargs.pop("embed_is_patch")
679
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
680
- raise ValueError("Incorrect type of embed_is_patch. "
681
- f"Got type: {type(embed_is_patch)}")
682
-
683
- embed_is_patch = flatten_bn(embed_is_patch)
684
-
640
if image_embeds is not None:
641
if not isinstance(image_embeds, (torch.Tensor, list)):
642
raise ValueError("Incorrect type of image embeddings. "
@@ -690,7 +645,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
645
return Idefics3ImageEmbeddingInputs(
646
type="image_embeds",
647
data=flatten_bn(image_embeds, concat=True),
693
- embed_is_patch=embed_is_patch,
648
)
649
650
if pixel_values is not None:
@@ -718,7 +672,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
672
pixel_values=self._validate_pixel_values(pixel_values),
673
pixel_attention_mask=pixel_attention_mask,
674
num_patches=num_patches,
721
- embed_is_patch=embed_is_patch,
675
)
676
677
raise AssertionError("This line should be unreachable.")
@@ -754,12 +707,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
707
if image_input is None:
708
return None
709
757
- image_features = self._process_image_input(image_input)
758
-
759
- return scatter_patch_features(
760
- image_features,
761
- image_input["embed_is_patch"],
762
- )
710
+ return self._process_image_input(image_input)
711
712
def get_input_embeddings(
713
self,
@@ -771,7 +719,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
719
inputs_embeds = merge_multimodal_embeddings(
720
input_ids,
721
inputs_embeds,
774
- select_patch_features(multimodal_embeddings),
722
+ multimodal_embeddings,
723
self.config.image_token_id,
724
)
725
return inputs_embeds
vllm/model_executor/models/internvl.py
+3
-40
@@ -39,7 +39,6 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
39
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
40
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41
maybe_prefix, merge_multimodal_embeddings)
42
-from .vision import scatter_patch_features, select_patch_features
42
43
IMG_START = '<img>'
44
IMG_END = '</img>'
@@ -60,14 +59,6 @@ class InternVLImagePixelInputs(TypedDict):
59
num_patches: torch.Tensor
60
"""Shape: `(batch_size * num_images)`"""
61
63
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
64
- """
65
- A boolean mask indicating which image embeddings correspond
66
- to patch tokens.
67
-
68
- Shape: `(batch_size * num_images, num_embeds)`
69
- """
70
-
62
63
class InternVLImageEmbeddingInputs(TypedDict):
64
type: Literal["image_embeds"]
@@ -419,24 +410,12 @@ class BaseInternVLProcessor(ABC):
410
torch.tensor([len(item) for item in pixel_values_lst]),
411
}
412
422
- tokenizer = self.tokenizer
423
- image_token_id = self.image_token_id
424
-
425
- embed_is_patch = list[torch.Tensor]()
426
-
413
for pixel_values in pixel_values_lst:
414
num_patches = pixel_values.shape[0]
415
feature_size = num_patches * self.num_image_token
416
417
image_repl = self.get_image_repl(feature_size, num_patches)
432
- feature_tokens = tokenizer.encode(image_repl.features,
433
- add_special_tokens=False)
434
-
418
text = [t.replace('<image>', image_repl.full, 1) for t in text]
436
- embed_is_patch.append(
437
- torch.tensor(feature_tokens) == image_token_id)
438
-
439
- image_inputs["embed_is_patch"] = embed_is_patch
419
420
text_inputs = self.tokenizer(text)
421
@@ -460,7 +439,7 @@ class InternVLProcessor(BaseInternVLProcessor):
439
repl_features = IMG_CONTEXT * feature_size
440
repl_full = IMG_START + repl_features + IMG_END
441
463
- return PromptUpdateDetails(full=repl_full, features=repl_features)
442
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
443
444
445
class BaseInternVLProcessingInfo(BaseProcessingInfo):
@@ -599,7 +578,6 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
578
pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
579
"image", image_num_patches),
580
image_num_patches=MultiModalFieldConfig.batched("image"),
602
- embed_is_patch=MultiModalFieldConfig.batched("image"),
581
image_embeds=MultiModalFieldConfig.batched("image"),
582
image_token_id=MultiModalFieldConfig.shared("image", num_images),
583
)
@@ -831,7 +809,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
809
self, **kwargs: object) -> Optional[InternVLImageInputs]:
810
pixel_values_flat = kwargs.pop("pixel_values_flat", None)
811
image_num_patches = kwargs.pop("image_num_patches", None)
834
- embed_is_patch = kwargs.pop("embed_is_patch", None)
812
image_embeds = kwargs.pop("image_embeds", None)
813
814
if pixel_values_flat is None and image_embeds is None:
@@ -860,20 +837,14 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
837
raise ValueError("Incorrect type of image_num_patches. "
838
f"Got type: {type(image_num_patches)}")
839
863
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
864
- raise ValueError("Incorrect type of embed_is_patch. "
865
- f"Got type: {type(embed_is_patch)}")
866
-
840
pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
841
image_num_patches = flatten_bn(image_num_patches, concat=True)
869
- embed_is_patch = flatten_bn(embed_is_patch)
842
843
return InternVLImagePixelInputs(
844
type="pixel_values",
845
pixel_values_flat=self._validate_pixel_values(
846
pixel_values_flat),
847
num_patches=image_num_patches,
876
- embed_is_patch=embed_is_patch,
848
)
849
850
raise AssertionError("This line should be unreachable.")
@@ -919,15 +890,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
890
if image_input is None:
891
return None
892
922
- image_features = self._process_image_input(image_input)
923
-
924
- if image_input["type"] != "pixel_values":
925
- return image_features
926
-
927
- return scatter_patch_features(
928
- image_features,
929
- image_input["embed_is_patch"],
930
- )
893
+ return self._process_image_input(image_input)
894
895
def get_input_embeddings(
896
self,
@@ -941,7 +904,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
904
inputs_embeds = merge_multimodal_embeddings(
905
input_ids,
906
inputs_embeds,
944
- select_patch_features(multimodal_embeddings),
907
+ multimodal_embeddings,
908
self.img_context_token_id,
909
)
910
return inputs_embeds
vllm/model_executor/models/llava.py
+6
-49
@@ -32,7 +32,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32
ImageSize, MultiModalDataItems)
33
from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
BaseProcessingInfo, ProcessingCache,
35
- PromptReplacement, PromptUpdate)
35
+ PromptReplacement, PromptUpdate,
36
+ PromptUpdateDetails)
37
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38
from vllm.sequence import IntermediateTensors
39
@@ -42,8 +43,7 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
43
from .siglip import SiglipVisionModel
44
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
45
maybe_prefix, merge_multimodal_embeddings)
45
-from .vision import (get_vision_encoder_info, scatter_patch_features,
46
- select_patch_features)
46
+from .vision import get_vision_encoder_info
47
48
49
class LlavaImagePixelInputs(TypedDict):
@@ -67,14 +67,6 @@ class PixtralHFImagePixelInputs(TypedDict):
67
in which case the data is passed as a list instead of a batched tensor.
68
"""
69
70
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
71
- """
72
- A boolean mask indicating which image embeddings correspond
73
- to patch tokens.
74
-
75
- Shape: `(batch_size * num_images, num_embeds)`
76
- """
77
-
70
71
class LlavaImageEmbeddingInputs(TypedDict):
72
type: Literal["image_embeds"]
@@ -343,23 +335,6 @@ class PixtralHFMultiModalProcessor(
335
for p, (h, w) in zip(pixel_values, image_sizes)
336
]
337
346
- hf_config = self.info.get_hf_config()
347
- vision_config = hf_config.vision_config
348
- assert isinstance(vision_config, PixtralVisionConfig)
349
- encoder_info = PixtralHFEncoderInfo(vision_config)
350
-
351
- tile_sizes = [
352
- encoder_info.get_patch_grid_size(
353
- image_width=pixel_value.shape[-1],
354
- image_height=pixel_value.shape[-2],
355
- ) for pixel_value in processed_outputs["pixel_values"]
356
- ]
357
- embed_is_patch = [
358
- torch.tensor(([True] * ncols + [False]) * nrows)
359
- for ncols, nrows in tile_sizes
360
- ]
361
- processed_outputs["embed_is_patch"] = embed_is_patch
362
-
338
return processed_outputs
339
340
def _get_mm_fields_config(
@@ -369,7 +344,6 @@ class PixtralHFMultiModalProcessor(
344
) -> Mapping[str, MultiModalFieldConfig]:
345
return dict(
346
pixel_values=MultiModalFieldConfig.batched("image"),
372
- embed_is_patch=MultiModalFieldConfig.batched("image"),
347
image_embeds=MultiModalFieldConfig.batched("image"),
348
)
349
@@ -404,7 +378,7 @@ class PixtralHFMultiModalProcessor(
378
tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
379
tokens[-1] = image_end_id
380
407
- return tokens
381
+ return PromptUpdateDetails.select_token_id(tokens, image_token_id)
382
383
return [
384
PromptReplacement(
@@ -612,17 +586,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
586
f"Got type: {type(pixel_values)}")
587
588
if self.config.vision_config.model_type == "pixtral":
615
- embed_is_patch = kwargs.pop("embed_is_patch")
616
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
617
- raise ValueError("Incorrect type of embed_is_patch. "
618
- f"Got type: {type(embed_is_patch)}")
619
-
620
- embed_is_patch = flatten_bn(embed_is_patch)
621
-
589
return PixtralHFImagePixelInputs(
590
type="pixel_values_pixtral",
591
pixel_values=flatten_bn(pixel_values),
625
- embed_is_patch=embed_is_patch,
592
)
593
594
return LlavaImagePixelInputs(
@@ -714,16 +680,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
680
if image_input is None:
681
return None
682
717
- image_features = self._process_image_input(image_input)
718
-
719
- if image_input["type"] != "pixel_values_pixtral":
720
- # The path is used for pixtral (V0 only) and llava (V0/V1)
721
- return image_features
722
-
723
- return scatter_patch_features(
724
- image_features,
725
- image_input["embed_is_patch"],
726
- )
683
+ return self._process_image_input(image_input)
684
685
def get_input_embeddings(
686
self,
@@ -735,7 +692,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
692
inputs_embeds = merge_multimodal_embeddings(
693
input_ids,
694
inputs_embeds,
738
- select_patch_features(multimodal_embeddings),
695
+ multimodal_embeddings,
696
self.config.image_token_index,
697
)
698
return inputs_embeds
vllm/model_executor/models/minicpmo.py
+9
-65
@@ -40,7 +40,8 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
40
DictEmbeddingItems, ModalityData,
41
ModalityDataItems, MultiModalDataItems,
42
MultiModalDataParser)
43
-from vllm.multimodal.processing import PromptReplacement, PromptUpdate
43
+from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
44
+ PromptUpdateDetails)
45
from vllm.multimodal.profiling import ProcessorInputs
46
47
from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,
@@ -50,7 +51,6 @@ from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,
51
_minicpmv_field_config)
52
from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn,
53
maybe_prefix)
53
-from .vision import scatter_patch_features
54
55
CPU_DEVICE = torch.device("cpu")
56
@@ -73,14 +73,6 @@ class MiniCPMOAudioFeatureInputs(TypedDict):
73
which equals to `audio_features.shape[-1]`
74
"""
75
76
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
77
- """
78
- A boolean mask indicating which audio embeddings correspond
79
- to patch tokens.
80
-
81
- Shape: `(batch_size * num_audios, num_embeds)`
82
- """
83
-
76
77
class MiniCPMOAudioEmbeddingInputs(TypedDict):
78
type: Literal["audio_embeds"]
@@ -93,14 +85,6 @@ class MiniCPMOAudioEmbeddingInputs(TypedDict):
85
Length of each slice may vary, so pass it as a list.
86
"""
87
96
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
97
- """
98
- A boolean mask indicating which audio embeddings correspond
99
- to patch tokens.
100
-
101
- Shape: `(batch_size * num_audios, num_embeds)`
102
- """
103
-
88
89
MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
90
MiniCPMOAudioEmbeddingInputs]
@@ -115,7 +99,6 @@ def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
99
audio_features=MultiModalFieldConfig.batched("audio"),
100
audio_feature_lens=MultiModalFieldConfig.batched("audio"),
101
audio_embeds=MultiModalFieldConfig.batched("audio"),
118
- audio_embed_is_patch=MultiModalFieldConfig.batched("audio"),
102
audio_token_id=MultiModalFieldConfig.shared("audio", num_audios),
103
)
104
@@ -197,8 +180,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
180
pool_step = self.get_default_audio_pool_step()
181
fbank_feat_in_chunk = 100
182
cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
200
- num_audio_tokens = (cnn_feat_in_chunk - pool_step) // pool_step + 1
201
- return num_audio_tokens + 2 # <audio>(<unk>*N)</audio>
183
+ return (cnn_feat_in_chunk - pool_step) // pool_step + 1
184
185
def get_max_audio_chunks_with_most_features(self) -> int:
186
return 30
@@ -209,8 +191,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
191
192
def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
193
sampling_rate = self.get_default_audio_sampling_rate()
212
- # exclude <audio> </audio>
213
- num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk() - 2
194
+ num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk()
195
return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1
196
197
def get_num_frames_with_most_features(
@@ -295,13 +276,6 @@ class MiniCPMOMultiModalProcessor(
276
277
if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
278
audio_inputs = {}
298
-
299
- audio_lens = [
300
- self.info.get_audio_len_by_num_chunks(
301
- sum(map(len,
302
- parsed_audios.get(i)["audio_embeds"])))
303
- for i in range(len(parsed_audios))
304
- ]
279
else:
280
audio_inputs = self._base_call_hf_processor(
281
prompts=[self.info.audio_pattern] * len(parsed_audios),
@@ -323,27 +297,7 @@ class MiniCPMOMultiModalProcessor(
297
]
298
audio_inputs["audio_features"] = unpadded_audio_features
299
326
- audio_lens = [
327
- parsed_audios.get_audio_length(i)
328
- for i in range(len(parsed_audios))
329
- ]
330
-
331
- audio_repl_features = [
332
- self.get_audio_prompt_texts(audio_len) for audio_len in audio_lens
333
- ]
334
-
300
tokenizer = self.info.get_tokenizer()
336
- audio_repls_feature_tokens = [
337
- tokenizer.encode(audio_repl, add_special_tokens=False)
338
- for audio_repl in audio_repl_features
339
- ]
340
-
341
- embed_is_patch = [
342
- self.get_embed_is_patch(audio_repl_tokens)
343
- for audio_repl_tokens in audio_repls_feature_tokens
344
- ]
345
- audio_inputs["audio_embed_is_patch"] = embed_is_patch
346
-
301
unk_token_id = tokenizer.get_vocab()["<unk>"]
302
audio_inputs["audio_token_id"] = torch.tensor(unk_token_id)
303
@@ -384,7 +338,10 @@ class MiniCPMOMultiModalProcessor(
338
else:
339
audio_len = audios.get_audio_length(item_idx)
340
387
- return self.get_audio_prompt_texts(audio_len)
341
+ return PromptUpdateDetails.select_text(
342
+ self.get_audio_prompt_texts(audio_len),
343
+ "<unk>",
344
+ )
345
346
return [
347
*base_updates,
@@ -713,13 +670,6 @@ class MiniCPMO(MiniCPMV2_6):
670
assert isinstance(audio_token_id, torch.Tensor)
671
self.mm_token_ids.add(audio_token_id.flatten().unique().item())
672
716
- audio_embed_is_patch = kwargs.pop("audio_embed_is_patch")
717
- if not isinstance(audio_embed_is_patch, (torch.Tensor, list)):
718
- raise ValueError("Incorrect type of audio_embed_is_patch. "
719
- f"Got type: {type(audio_embed_is_patch)}")
720
-
721
- audio_embed_is_patch = flatten_bn(audio_embed_is_patch)
722
-
673
if audio_embeds is not None:
674
if not isinstance(audio_embeds, (torch.Tensor, list)):
675
raise ValueError("Incorrect type of audio_embeds. "
@@ -730,7 +680,6 @@ class MiniCPMO(MiniCPMV2_6):
680
return MiniCPMOAudioEmbeddingInputs(
681
type="audio_embeds",
682
audio_embeds=audio_embeds_flat,
733
- embed_is_patch=audio_embed_is_patch,
683
)
684
685
if not isinstance(audio_features, (torch.Tensor, list)):
@@ -749,7 +698,6 @@ class MiniCPMO(MiniCPMV2_6):
698
type="audio_features",
699
audio_features=audio_features_flat,
700
audio_feature_lens=audio_feature_lens_flat,
752
- embed_is_patch=audio_embed_is_patch,
701
)
702
703
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
@@ -781,10 +729,6 @@ class MiniCPMO(MiniCPMV2_6):
729
if modality == "audios":
730
audio_input = modalities["audios"]
731
audio_features = self._process_audio_input(audio_input)
784
- multimodal_embeddings += tuple(
785
- scatter_patch_features(
786
- audio_features,
787
- audio_input["embed_is_patch"],
788
- ))
732
+ multimodal_embeddings += tuple(audio_features)
733
734
return multimodal_embeddings
vllm/model_executor/models/minicpmv.py
+41
-101
@@ -56,7 +56,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
56
VideoItem, VideoProcessorItems)
57
from vllm.multimodal.processing import (BaseMultiModalProcessor,
58
BaseProcessingInfo, PromptReplacement,
59
- PromptUpdate)
59
+ PromptUpdate, PromptUpdateDetails)
60
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
61
from vllm.platforms import current_platform
62
from vllm.sequence import IntermediateTensors
@@ -67,7 +67,6 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
67
SupportsMultiModal, SupportsPP)
68
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
69
merge_multimodal_embeddings)
70
-from .vision import scatter_patch_features, select_patch_features
70
71
# For profile run
72
_MAX_FRAMES_PER_VIDEO = 16
@@ -90,14 +89,6 @@ class MiniCPMVImagePixelInputs(TypedDict):
89
This should be in `(height, width)` format.
90
"""
91
93
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
94
- """
95
- A boolean mask indicating which image embeddings correspond
96
- to patch tokens.
97
-
98
- Shape: `(batch_size * num_images, num_embeds)`
99
- """
100
-
92
num_slices: torch.Tensor
93
"""Shape: `(batch_size * num_images)`"""
94
@@ -112,14 +103,6 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
103
instead of a batched tensor.
104
"""
105
115
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
116
- """
117
- A boolean mask indicating which image embeddings correspond
118
- to patch tokens.
119
-
120
- Shape: `(batch_size * num_images, num_embeds)`
121
- """
122
-
106
107
MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
108
MiniCPMVImageEmbeddingInputs]
@@ -245,12 +228,10 @@ def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
228
image_sizes=MultiModalFieldConfig.batched("image"),
229
tgt_sizes=MultiModalFieldConfig.batched("image"),
230
image_embeds=MultiModalFieldConfig.batched("image"),
248
- embed_is_patch=MultiModalFieldConfig.batched("image"),
231
video_pixel_values=MultiModalFieldConfig.batched("video"),
232
video_image_sizes=MultiModalFieldConfig.batched("video"),
233
video_tgt_sizes=MultiModalFieldConfig.batched("video"),
234
video_embeds=MultiModalFieldConfig.batched("video"),
253
- video_embed_is_patch=MultiModalFieldConfig.batched("video"),
235
image_token_id=MultiModalFieldConfig.shared("image", num_images),
236
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
237
)
@@ -398,22 +379,43 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
379
use_image_id=use_image_id,
380
)
381
382
+ def get_sliced_grid(
383
+ self,
384
+ image_size: ImageSize,
385
+ # For MiniCPM V/O 2.6
386
+ max_slice_nums: Optional[int] = None,
387
+ ) -> Optional[tuple[int, int]]:
388
+ image_processor = self.get_image_processor()
389
+ version = self.get_model_version()
390
+
391
+ if version == (2, 0) or version == (2, 5):
392
+ return image_processor.get_sliced_grid(image_size)
393
+
394
+ if max_slice_nums is None:
395
+ max_slice_nums = image_processor.max_slice_nums
396
+
397
+ return image_processor.get_sliced_grid(
398
+ image_size,
399
+ max_slice_nums=max_slice_nums,
400
+ )
401
+
402
def get_num_image_tokens(
403
self,
404
image_size: ImageSize,
405
max_slice_nums: Optional[int] = None,
405
- use_image_id: bool = True,
406
) -> int:
407
- tokenizer = self.get_tokenizer()
408
- image_placeholders = self.get_slice_image_placeholder(
407
+ image_processor = self.get_image_processor()
408
+
409
+ grid = self.get_sliced_grid(
410
image_size,
411
max_slice_nums=max_slice_nums,
411
- use_image_id=use_image_id,
412
)
413
- image_token_ids = tokenizer.encode(image_placeholders,
414
- add_special_tokens=False)
413
+ if grid is None:
414
+ ncols = nrows = 0
415
+ else:
416
+ ncols, nrows = grid
417
416
- return len(image_token_ids)
418
+ return (ncols * nrows + 1) * image_processor.image_feature_size
419
420
def get_max_image_tokens(self) -> int:
421
image_size = self.get_image_size_with_most_features()
@@ -433,7 +435,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
435
return self.get_num_image_tokens(
436
frame_size,
437
max_slice_nums=self.get_video_max_slice_num(),
436
- use_image_id=False,
438
)
439
440
def get_max_video_tokens(
@@ -539,14 +540,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
540
use_image_id=False,
541
) * num_frames
542
542
- def get_embed_is_patch(
543
- self,
544
- input_ids: list[int],
545
- ) -> torch.Tensor:
546
- tokenizer = self.info.get_tokenizer()
547
- unk_token_id = tokenizer.get_vocab()["<unk>"]
548
- return torch.tensor(input_ids) == unk_token_id
549
-
543
def process_images(
544
self,
545
mm_data: Mapping[str, object],
@@ -570,26 +563,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
563
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
564
)
565
573
- image_sizes = [
574
- parsed_images.get_image_size(i) for i in range(len(parsed_images))
575
- ]
576
- image_repl_features = [
577
- self.get_image_prompt_texts(size, idx)
578
- for idx, size in enumerate(image_sizes)
579
- ]
580
-
566
tokenizer = self.info.get_tokenizer()
582
- image_repls_feature_tokens = [
583
- tokenizer.encode(image_repl, add_special_tokens=False)
584
- for image_repl in image_repl_features
585
- ]
586
-
587
- embed_is_patch = [
588
- self.get_embed_is_patch(image_repl_tokens)
589
- for image_repl_tokens in image_repls_feature_tokens
590
- ]
591
- image_inputs["embed_is_patch"] = embed_is_patch
592
-
567
unk_token_id = tokenizer.get_vocab()["<unk>"]
568
image_inputs["image_token_id"] = torch.tensor(unk_token_id)
569
@@ -625,31 +599,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
599
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
600
)
601
628
- frame_sizes = [
629
- parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))
630
- ]
631
- num_frames = [
632
- parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))
633
- ]
634
- video_repl_features = [
635
- self.get_video_prompt_texts(size, nframes)
636
- for size, nframes in zip(frame_sizes, num_frames)
637
- ]
638
-
639
- tokenizer = self.info.get_tokenizer()
640
- video_repls_feature_tokens = [
641
- tokenizer.encode(video_repl, add_special_tokens=False)
642
- for video_repl in video_repl_features
643
- ]
644
-
645
- embed_is_patch = [
646
- self.get_embed_is_patch(video_repl_tokens)
647
- for video_repl_tokens in video_repls_feature_tokens
648
- ]
649
- video_inputs["embed_is_patch"] = embed_is_patch
650
-
602
video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
603
604
+ tokenizer = self.info.get_tokenizer()
605
unk_token_id = tokenizer.get_vocab()["<unk>"]
606
video_inputs["video_token_id"] = torch.tensor(unk_token_id)
607
@@ -740,7 +692,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
692
693
image_size = images.get_image_size(item_idx)
694
743
- return self.get_image_prompt_texts(image_size, item_idx)
695
+ return PromptUpdateDetails.select_text(
696
+ self.get_image_prompt_texts(image_size, item_idx),
697
+ "<unk>",
698
+ )
699
700
def get_video_replacement(item_idx: int):
701
videos = mm_items.get_items(
@@ -749,7 +704,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
704
frame_size = videos.get_frame_size(item_idx)
705
num_frames = videos.get_num_frames(item_idx)
706
752
- return self.get_video_prompt_texts(frame_size, num_frames)
707
+ return PromptUpdateDetails.select_text(
708
+ self.get_video_prompt_texts(frame_size, num_frames),
709
+ "<unk>",
710
+ )
711
712
get_replacement = {
713
"image": get_image_replacement,
@@ -832,14 +790,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
790
assert isinstance(image_token_id, torch.Tensor)
791
self.mm_token_ids.add(image_token_id.flatten().unique().item())
792
835
- embed_is_patch = kwargs.pop("embed_is_patch")
836
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
837
- raise ValueError(
838
- f"Incorrect type of embed_is_patch for {modality=}. "
839
- f"Got type: {type(embed_is_patch)}")
840
-
841
- embed_is_patch = flatten_bn(embed_is_patch)
842
-
793
if image_embeds is not None:
794
if not isinstance(image_embeds, (torch.Tensor, list)):
795
raise ValueError(
@@ -851,7 +801,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
801
return MiniCPMVImageEmbeddingInputs(
802
type="image_embeds",
803
image_embeds=image_embeds_flat,
854
- embed_is_patch=embed_is_patch,
804
)
805
806
if not isinstance(pixel_values, (torch.Tensor, list)):
@@ -879,7 +828,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
828
type="pixel_values",
829
pixel_values=pixel_values_flat,
830
tgt_sizes=tgt_sizes_flat,
882
- embed_is_patch=embed_is_patch,
831
num_slices=num_slices_flat,
832
)
833
@@ -936,19 +884,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
884
if modality == "images":
885
image_input = modalities["images"]
886
image_features = self._process_vision_input(image_input)
939
- multimodal_embeddings += tuple(
940
- scatter_patch_features(
941
- image_features,
942
- image_input["embed_is_patch"],
943
- ))
887
+ multimodal_embeddings += tuple(image_features)
888
if modality == "videos":
889
video_input = modalities["videos"]
890
video_features = self._process_vision_input(video_input)
947
- multimodal_embeddings += tuple(
948
- scatter_patch_features(
949
- video_features,
950
- video_input["embed_is_patch"],
951
- ))
891
+ multimodal_embeddings += tuple(video_features)
892
893
return multimodal_embeddings
894
@@ -971,7 +911,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
911
inputs_embeds = merge_multimodal_embeddings(
912
input_ids,
913
inputs_embeds,
974
- select_patch_features(multimodal_embeddings),
914
+ multimodal_embeddings,
915
list(self.mm_token_ids),
916
)
917
return inputs_embeds
vllm/model_executor/models/mistral3.py
+6
-44
@@ -27,7 +27,8 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
27
MultiModalDataItems)
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
BaseProcessingInfo, ProcessingCache,
30
- PromptReplacement, PromptUpdate)
30
+ PromptReplacement, PromptUpdate,
31
+ PromptUpdateDetails)
32
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
33
from vllm.sequence import IntermediateTensors
34
@@ -35,8 +36,7 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
36
from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
37
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38
maybe_prefix, merge_multimodal_embeddings)
38
-from .vision import (get_vision_encoder_info, scatter_patch_features,
39
- select_patch_features)
39
+from .vision import get_vision_encoder_info
40
41
42
class Mistral3ImagePixelInputs(TypedDict):
@@ -49,14 +49,6 @@ class Mistral3ImagePixelInputs(TypedDict):
49
in which case the data is passed as a list instead of a batched tensor.
50
"""
51
52
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
53
- """
54
- A boolean mask indicating which image embeddings correspond
55
- to patch tokens.
56
-
57
- Shape: `(batch_size, num_images, num_embeds)`
58
- """
59
-
52
53
class Mistral3PatchMerger(nn.Module):
54
"""
@@ -266,23 +258,6 @@ class Mistral3MultiModalProcessor(
258
p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
259
]
260
269
- hf_config = self.info.get_hf_config()
270
- vision_config = hf_config.vision_config
271
- assert isinstance(vision_config, PixtralVisionConfig)
272
- encoder_info = PixtralHFEncoderInfo(vision_config)
273
-
274
- tile_sizes = [
275
- encoder_info.get_patch_grid_size(
276
- image_width=pixel_value.shape[-1],
277
- image_height=pixel_value.shape[-2],
278
- ) for pixel_value in processed_outputs["pixel_values"]
279
- ]
280
- embed_is_patch = [
281
- torch.tensor(([True] * ncols + [False]) * nrows)
282
- for ncols, nrows in tile_sizes
283
- ]
284
- processed_outputs["embed_is_patch"] = embed_is_patch
285
-
261
return processed_outputs
262
263
def _get_mm_fields_config(
@@ -292,7 +267,6 @@ class Mistral3MultiModalProcessor(
267
) -> Mapping[str, MultiModalFieldConfig]:
268
return dict(
269
pixel_values=MultiModalFieldConfig.batched("image"),
295
- embed_is_patch=MultiModalFieldConfig.batched("image"),
270
image_embeds=MultiModalFieldConfig.batched("image"),
271
)
272
@@ -327,7 +301,7 @@ class Mistral3MultiModalProcessor(
301
tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
302
tokens[-1] = image_end_id
303
330
- return tokens
304
+ return PromptUpdateDetails.select_token_id(tokens, image_token_id)
305
306
return [
307
PromptReplacement(
@@ -418,8 +392,6 @@ def init_vision_tower_for_llava(
392
)
393
394
421
-# TODO(mgoin): Support V1, there are issues with image batching/chunking
422
-# that need to be resolved first.
395
@MULTIMODAL_REGISTRY.register_processor(
396
_build_mistral3_processor,
397
info=_build_mistral3_info,
@@ -509,16 +481,9 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
481
raise ValueError("Incorrect type of pixel values. "
482
f"Got type: {type(pixel_values)}")
483
512
- assert self.config.vision_config.model_type == "pixtral"
513
- embed_is_patch = kwargs.pop("embed_is_patch")
514
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
515
- raise ValueError("Incorrect type of embed_is_patch. "
516
- f"Got type: {type(embed_is_patch)}")
517
-
484
return Mistral3ImagePixelInputs(
485
type="pixel_values_pixtral",
486
pixel_values=flatten_bn(pixel_values),
521
- embed_is_patch=flatten_bn(embed_is_patch),
487
)
488
489
def _process_image_input(
@@ -557,10 +522,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
522
523
vision_embeddings = self._process_image_input(image_input)
524
560
- return scatter_patch_features(
561
- vision_embeddings,
562
- image_input["embed_is_patch"],
563
- )
525
+ return vision_embeddings
526
527
def get_input_embeddings(
528
self,
@@ -572,7 +534,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
534
inputs_embeds = merge_multimodal_embeddings(
535
input_ids,
536
inputs_embeds,
575
- select_patch_features(multimodal_embeddings),
537
+ multimodal_embeddings,
538
self.config.image_token_index,
539
)
540
return inputs_embeds
vllm/model_executor/models/molmo.py
+13
-61
@@ -46,7 +46,8 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
46
MultiModalDataItems)
47
from vllm.multimodal.processing import (BaseMultiModalProcessor,
48
BaseProcessingInfo, PromptIndexTargets,
49
- PromptInsertion, PromptUpdate)
49
+ PromptInsertion, PromptUpdate,
50
+ PromptUpdateDetails)
51
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52
from vllm.sequence import IntermediateTensors
53
@@ -56,7 +57,6 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
57
is_pp_missing_parameter,
58
make_empty_intermediate_tensors_factory, make_layers,
59
maybe_prefix, merge_multimodal_embeddings)
59
-from .vision import scatter_patch_features, select_patch_features
60
61
# TODO: hard-coded for now. Consider making it configurable.
62
VIT_LAYERS = [-2, -9]
@@ -84,14 +84,6 @@ class MolmoImageInputs(TypedDict):
84
Shape: `(batch_size * num_images, num_crops, num_patch)`
85
"""
86
87
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
88
- """
89
- A boolean mask indicating which image embeddings correspond
90
- to patch tokens.
91
-
92
- Shape: `(batch_size * num_images, num_embeds)`
93
- """
94
-
87
num_crops: torch.Tensor
88
"""Shape: `(batch_size * num_images)`"""
89
@@ -1146,30 +1138,6 @@ class MolmoProcessorWrapper:
1138
if image_input_idx is not None:
1139
feat_is_patch = image_input_idx >= 0
1140
1149
- input_is_embed = torch.isin(
1150
- input_ids,
1151
- torch.tensor([
1152
- self.image_patch_id,
1153
- self.im_col_id,
1154
- self.im_start_id,
1155
- self.im_end_id,
1156
- ]),
1157
- )
1158
- embed_ids = input_ids[input_is_embed]
1159
- embed_is_patch = embed_ids == self.image_patch_id
1160
- assert embed_is_patch.sum() == feat_is_patch.sum()
1161
-
1162
- # image_tokens = extra_joint + joint
1163
- # Both `extra_joint` and `joint` have `im_start_id` and `im_end_id`
1164
- embed_start = torch.nonzero(embed_ids == self.im_start_id)[::2, 0]
1165
- embed_end = torch.nonzero(embed_ids == self.im_end_id)[1::2, 0]
1166
- assert len(embed_start) == len(embed_end) == len(images)
1167
-
1168
- embed_is_patch = [
1169
- embed_is_patch[start:end + 1]
1170
- for start, end in zip(embed_start, embed_end)
1171
- ]
1172
-
1141
tilings = [
1142
self.select_tiling(
1143
image_width=image.size[0],
@@ -1181,7 +1149,6 @@ class MolmoProcessorWrapper:
1149
assert num_crops.sum() == len(feat_is_patch)
1150
1151
outputs["feat_is_patch"] = feat_is_patch
1184
- outputs["embed_is_patch"] = embed_is_patch
1152
outputs["num_crops"] = num_crops
1153
outputs["img_patch_id"] = self.image_patch_id
1154
@@ -1220,17 +1187,13 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1187
)
1188
pooling_size = processor.pooling_size
1189
1223
- base_image_input_size = processor.base_image_input_size
1224
- base_image_input_d = processor.image_patch_size
1225
-
1226
- crop_patches = base_image_input_size[0] // base_image_input_d
1190
+ image_token_length_w = processor.image_token_length_w
1191
+ image_token_length_h = processor.image_token_length_h
1192
1228
- per_row = ncols // pooling_size + 1
1229
- joint = per_row * (nrows // pooling_size) + 2
1230
- image_token_length = (crop_patches + pooling_size - 1) // pooling_size
1231
- resize = (image_token_length + 1) * image_token_length + 2
1193
+ extra = image_token_length_w * image_token_length_h
1194
+ joint = ((ncols + 1) // pooling_size) * ((nrows + 1) // pooling_size)
1195
1233
- return resize + joint
1196
+ return extra + joint
1197
1198
def get_max_image_tokens(self) -> int:
1199
target_width, target_height = self.get_image_size_with_most_features()
@@ -1328,7 +1291,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1291
"image", num_crops),
1292
feat_is_patch=MultiModalFieldConfig.flat_from_sizes(
1293
"image", num_crops),
1331
- embed_is_patch=MultiModalFieldConfig.batched("image"),
1294
num_crops=MultiModalFieldConfig.batched("image"),
1295
img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1296
)
@@ -1368,8 +1330,10 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1330
joint = ([img_start_id] + joint_row *
1331
((nrows + 1) // pooling_size) + [img_end_id])
1332
1371
- image_tokens = extra_joint + joint
1372
- return image_tokens
1333
+ return PromptUpdateDetails.select_token_id(
1334
+ extra_joint + joint,
1335
+ embed_token_id=img_patch_id,
1336
+ )
1337
1338
return [
1339
PromptInsertion(
@@ -1475,11 +1439,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1439
raise ValueError("Incorrect type of feat_is_patch. "
1440
f"Got type: {type(feat_is_patch)}")
1441
1478
- embed_is_patch = kwargs.pop("embed_is_patch", None)
1479
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
1480
- raise ValueError("Incorrect type of embed_is_patch. "
1481
- f"Got type: {type(embed_is_patch)}")
1482
-
1442
num_crops = kwargs.pop("num_crops", None)
1443
if not isinstance(num_crops, (torch.Tensor, list)):
1444
raise ValueError("Incorrect type of num_crops. "
@@ -1491,14 +1450,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1450
f"Got type: {type(img_patch_id)}")
1451
self.img_patch_id = img_patch_id.flatten().unique().item()
1452
1494
- embed_is_patch = flatten_bn(embed_is_patch)
1453
num_crops = flatten_bn(num_crops, concat=True)
1454
1455
return MolmoImageInputs(
1456
images=images,
1457
image_masks=image_masks,
1458
feat_is_patch=feat_is_patch,
1501
- embed_is_patch=embed_is_patch,
1459
num_crops=num_crops,
1460
)
1461
@@ -1537,12 +1494,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1494
if image_input is None:
1495
return None
1496
1540
- image_features = self._process_image_input(image_input)
1541
-
1542
- return scatter_patch_features(
1543
- image_features,
1544
- image_input["embed_is_patch"],
1545
- )
1497
+ return self._process_image_input(image_input)
1498
1499
def get_input_embeddings(
1500
self,
@@ -1556,7 +1508,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1508
inputs_embeds = merge_multimodal_embeddings(
1509
input_ids,
1510
inputs_embeds,
1559
- select_patch_features(multimodal_embeddings),
1511
+ multimodal_embeddings,
1512
self.img_patch_id,
1513
)
1514
return inputs_embeds
vllm/model_executor/models/nvlm_d.py
+2
-30
@@ -57,7 +57,7 @@ class NVLMProcessor(BaseInternVLProcessor):
57
# when trying to find "<tile" as a subsequence of "<Image><tile"
58
repl = "<Image>" + features + "</Image>"
59
60
- return PromptUpdateDetails(full=repl, features=repl)
60
+ return PromptUpdateDetails.select_text(repl, IMG_PAD)
61
62
63
class NVLMProcessingInfo(BaseInternVLProcessingInfo):
@@ -84,31 +84,6 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):
84
**kwargs,
85
)
86
87
- def get_max_image_tokens(self) -> int:
88
- hf_processor = self.get_hf_processor()
89
- tokenizer = hf_processor.tokenizer
90
-
91
- max_num_patches = hf_processor.max_dynamic_patch
92
- # we need +1 here because max_dynamic_patch in config doesn't
93
- # include the thumbnail patch
94
- tile_pos_identifiers = [
95
- f"<tile_{i+1}>" for i in range(max_num_patches)
96
- ]
97
- if hf_processor.use_thumbnail and max_num_patches != 1:
98
- tile_pos_identifiers += ["<tile_global_thumbnail>"]
99
-
100
- # "<Image><tile" is tokenized as ["<Image", "><", "tile"]
101
- # so we include <tile_1> in the start_str
102
- start_str = "<Image>" + tile_pos_identifiers.pop(0)
103
- end_str = "</Image>"
104
- start_token_len = len(tokenizer.encode(start_str))
105
- end_token_len = len(tokenizer.encode(end_str))
106
- tile_token_len = sum(
107
- len(tokenizer.encode(identifier))
108
- for identifier in tile_pos_identifiers)
109
- non_image_tokens_num = start_token_len + end_token_len + tile_token_len
110
- return super().get_max_image_tokens() + non_image_tokens_num
111
-
87
88
class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
89
@@ -177,10 +152,7 @@ class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
152
153
repl = hf_processor.get_image_repl(feature_size, num_patches)
154
180
- return PromptUpdateDetails(
181
- full=repl.full + "\n",
182
- features=repl.features + "\n",
183
- )
155
+ return PromptUpdateDetails.select_text(repl.full + "\n", IMG_PAD)
156
157
# See note in dummy data regarding why we have the extra newline
158
return [
vllm/model_executor/models/paligemma.py
+3
-3
@@ -162,9 +162,9 @@ class PaliGemmaMultiModalProcessor(
162
modality="image",
163
target=PromptIndexTargets.prefix(
164
[bos_token_id] if tokenizer.add_bos_token else []),
165
- insertion=PromptUpdateDetails(
166
- full=image_tokens + [bos_token_id],
167
- features=image_tokens,
165
+ insertion=PromptUpdateDetails.select_token_id(
166
+ image_tokens + [bos_token_id],
167
+ embed_token_id=image_token_id,
168
),
169
)
170
]
vllm/model_executor/models/phi3v.py
+3
-8
@@ -40,8 +40,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
40
from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
BaseProcessingInfo, BoundPromptUpdate,
42
PlaceholderFeaturesInfo,
43
- PromptReplacement, PromptUpdate,
44
- PromptUpdateDetails)
43
+ PromptReplacement, PromptUpdate)
44
# yapf: enable
45
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46
from vllm.sequence import IntermediateTensors
@@ -443,12 +442,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
442
processor=hf_processor,
443
)
444
446
- image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
447
-
448
- return PromptUpdateDetails(
449
- full=image_tokens,
450
- features=image_tokens,
451
- )
445
+ return [_IMAGE_TOKEN_ID] * num_image_tokens
446
447
num_images = mm_items.get_count("image", strict=False)
448
@@ -517,6 +511,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
511
item_idx=p.item_idx,
512
start_idx=p.start_idx - 1,
513
tokens=p.tokens,
514
+ is_embed=p.is_embed,
515
) for p in ps
516
]
517
for modality, ps in placeholders.items()
vllm/model_executor/models/pixtral.py
+8
-40
@@ -37,7 +37,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37
MultiModalDataItems)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
BaseProcessingInfo, PromptReplacement,
40
- PromptUpdate)
40
+ PromptUpdate, PromptUpdateDetails)
41
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42
from vllm.sequence import IntermediateTensors
43
from vllm.transformers_utils.tokenizer import (MistralTokenizer,
@@ -46,8 +46,7 @@ from vllm.transformers_utils.tokenizer import (MistralTokenizer,
46
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
47
from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,
48
merge_multimodal_embeddings)
49
-from .vision import (VisionEncoderInfo, resolve_visual_encoder_outputs,
50
- scatter_patch_features, select_patch_features)
49
+from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
50
51
try:
52
from xformers import ops as xops
@@ -68,14 +67,6 @@ class PixtralImagePixelInputs(TypedDict):
67
The result of stacking :attr:`ImageEncoding.tokens` from each prompt.
68
"""
69
71
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
72
- """
73
- A boolean mask indicating which image embeddings correspond
74
- to patch tokens.
75
-
76
- Shape: `(batch_size * num_images, num_embeds)`
77
- """
78
-
70
71
class PixtralProcessorAdapter:
72
"""
@@ -144,11 +135,8 @@ class PixtralProcessorAdapter:
135
"For more info, see: "
136
"https://github.com/vllm-project/vllm/issues/8411.")
137
147
- image_token_id = self.image_token_id
148
-
138
images_processed = list[torch.Tensor]()
139
images_tokens = list[torch.Tensor]()
151
- images_embed_is_patch = list[torch.Tensor]()
140
141
for image in images:
142
image_inputs = self.image_processor(ImageChunk(image=image))
@@ -157,12 +145,10 @@ class PixtralProcessorAdapter:
145
146
images_processed.append(image_processed)
147
images_tokens.append(image_tokens)
160
- images_embed_is_patch.append(image_tokens == image_token_id)
148
149
return {
150
"input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),
151
"images": images_processed,
165
- "embed_is_patch": images_embed_is_patch,
152
}
153
154
@@ -213,7 +199,7 @@ class PixtralProcessingInfo(BaseProcessingInfo):
199
ncols, nrows = processor.image_processor._image_to_num_tokens(
200
Image.new("RGB", (image_width, image_height)))
201
216
- return (ncols + 1) * nrows
202
+ return ncols * nrows
203
204
def get_image_size_with_most_features(self) -> ImageSize:
205
image_processor = self.get_hf_processor().image_processor
@@ -263,10 +249,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
249
hf_inputs: Mapping[str, NestedTensors],
250
hf_processor_mm_kwargs: Mapping[str, object],
251
) -> Mapping[str, MultiModalFieldConfig]:
266
- return dict(
267
- images=MultiModalFieldConfig.batched("image"),
268
- embed_is_patch=MultiModalFieldConfig.batched("image"),
269
- )
252
+ return dict(images=MultiModalFieldConfig.batched("image"))
253
254
def _get_prompt_updates(
255
self,
@@ -290,7 +273,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
273
tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
274
tokens[-1] = image_end_id
275
293
- return tokens
276
+ return PromptUpdateDetails.select_token_id(tokens, image_token_id)
277
278
return [
279
PromptReplacement(
@@ -381,17 +364,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
364
raise ValueError("Incorrect type of images. "
365
f"Got type: {type(images)}")
366
384
- embed_is_patch = kwargs.pop("embed_is_patch")
385
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
386
- raise ValueError("Incorrect type of embed_is_patch. "
387
- f"Got type: {type(embed_is_patch)}")
388
-
389
- embed_is_patch = flatten_bn(embed_is_patch)
390
-
367
return PixtralImagePixelInputs(
368
type="pixel_values",
369
images=flatten_bn(images),
394
- embed_is_patch=embed_is_patch,
370
)
371
372
def _process_image_input(
@@ -427,12 +402,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
402
if image_input is None:
403
return None
404
430
- image_features = self._process_image_input(image_input)
431
-
432
- return scatter_patch_features(
433
- image_features,
434
- image_input["embed_is_patch"],
435
- )
405
+ return self._process_image_input(image_input)
406
407
def get_input_embeddings(
408
self,
@@ -444,7 +414,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
414
inputs_embeds = merge_multimodal_embeddings(
415
input_ids,
416
inputs_embeds,
447
- select_patch_features(multimodal_embeddings),
417
+ multimodal_embeddings,
418
self.vision_args.image_token_id,
419
)
420
return inputs_embeds
@@ -963,9 +933,7 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
933
image_width=image_width,
934
image_height=image_height,
935
)
966
-
967
- # Consider the image_break_token
968
- return (ncols + 1) * nrows
936
+ return ncols * nrows
937
938
def get_max_image_tokens(self) -> int:
939
image_size = self.get_image_size()
vllm/model_executor/models/qwen2_audio.py
+3
-3
@@ -229,9 +229,9 @@ class Qwen2AudioMultiModalProcessor(
229
230
audio_tokens = [audio_token_id] * num_features
231
232
- return PromptUpdateDetails(
233
- full=[audio_bos_id] + audio_tokens + [audio_eos_id],
234
- features=audio_tokens,
232
+ return PromptUpdateDetails.select_token_id(
233
+ [audio_bos_id] + audio_tokens + [audio_eos_id],
234
+ embed_token_id=audio_token_id,
235
)
236
237
return [
vllm/model_executor/models/qwen_vl.py
+3
-3
@@ -647,9 +647,9 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
647
PromptReplacement(
648
modality="image",
649
target=[img_start_id, img_end_id],
650
- replacement=PromptUpdateDetails(
651
- full=[img_start_id] + image_tokens + [img_end_id],
652
- features=image_tokens,
650
+ replacement=PromptUpdateDetails.select_token_id(
651
+ [img_start_id] + image_tokens + [img_end_id],
652
+ embed_token_id=img_pad_id,
653
),
654
)
655
]
vllm/model_executor/models/skyworkr1v.py
+3
-39
@@ -40,7 +40,6 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
40
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
41
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
42
maybe_prefix, merge_multimodal_embeddings)
43
-from .vision import scatter_patch_features, select_patch_features
43
44
IMG_START = '<img>'
45
IMG_END = '</img>'
@@ -61,14 +60,6 @@ class SkyworkR1VImagePixelInputs(TypedDict):
60
num_patches: torch.Tensor
61
"""Shape: `(batch_size * num_images)`"""
62
64
- embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
65
- """
66
- A boolean mask indicating which image embeddings correspond
67
- to patch tokens.
68
-
69
- Shape: `(batch_size * num_images, num_embeds)`
70
- """
71
-
63
64
class SkyworkR1VImageEmbeddingInputs(TypedDict):
65
type: Literal["image_embeds"]
@@ -419,24 +410,13 @@ class BaseSkyworkR1VProcessor(ABC):
410
torch.tensor([len(item) for item in pixel_values_lst]),
411
}
412
422
- tokenizer = self.tokenizer
423
- image_token_id = self.image_token_id
424
-
425
- embed_is_patch = list[torch.Tensor]()
426
-
413
for pixel_values in pixel_values_lst:
414
num_patches = pixel_values.shape[0]
415
feature_size = num_patches * self.num_image_token
416
417
image_repl = self.get_image_repl(feature_size, num_patches)
432
- feature_tokens = tokenizer.encode(image_repl.features,
433
- add_special_tokens=False)
418
419
text = [t.replace('<image>', image_repl.full, 1) for t in text]
436
- embed_is_patch.append(
437
- torch.tensor(feature_tokens) == image_token_id)
438
-
439
- image_inputs["embed_is_patch"] = embed_is_patch
420
421
text_inputs = self.tokenizer(text)
422
@@ -460,7 +440,7 @@ class SkyworkR1VProcessor(BaseSkyworkR1VProcessor):
440
repl_features = IMG_CONTEXT * feature_size
441
repl_full = IMG_START + repl_features + IMG_END
442
463
- return PromptUpdateDetails(full=repl_full, features=repl_features)
443
+ return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
444
445
446
class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
@@ -599,7 +579,6 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
579
pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
580
"image", image_num_patches),
581
image_num_patches=MultiModalFieldConfig.batched("image"),
602
- embed_is_patch=MultiModalFieldConfig.batched("image"),
582
image_embeds=MultiModalFieldConfig.batched("image"),
583
image_token_id=MultiModalFieldConfig.shared("image", num_images),
584
)
@@ -835,7 +814,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
814
self, **kwargs: object) -> Optional[SkyworkR1VImageInputs]:
815
pixel_values_flat = kwargs.pop("pixel_values_flat", None)
816
image_num_patches = kwargs.pop("image_num_patches", None)
838
- embed_is_patch = kwargs.pop("embed_is_patch", None)
817
image_embeds = kwargs.pop("image_embeds", None)
818
819
if pixel_values_flat is None and image_embeds is None:
@@ -864,20 +842,14 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
842
raise ValueError("Incorrect type of image_num_patches. "
843
f"Got type: {type(image_num_patches)}")
844
867
- if not isinstance(embed_is_patch, (torch.Tensor, list)):
868
- raise ValueError("Incorrect type of embed_is_patch. "
869
- f"Got type: {type(embed_is_patch)}")
870
-
845
pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
846
image_num_patches = flatten_bn(image_num_patches, concat=True)
873
- embed_is_patch = flatten_bn(embed_is_patch)
847
848
return SkyworkR1VImagePixelInputs(
849
type="pixel_values",
850
pixel_values_flat=self._validate_pixel_values(
851
pixel_values_flat),
852
num_patches=image_num_patches,
880
- embed_is_patch=embed_is_patch,
853
)
854
855
raise AssertionError("This line should be unreachable.")
@@ -923,15 +895,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
895
if image_input is None:
896
return None
897
926
- image_features = self._process_image_input(image_input)
927
-
928
- if image_input["type"] != "pixel_values":
929
- return image_features
930
-
931
- return scatter_patch_features(
932
- image_features,
933
- image_input["embed_is_patch"],
934
- )
898
+ return self._process_image_input(image_input)
899
900
def get_input_embeddings(
901
self,
@@ -945,7 +909,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
909
inputs_embeds = merge_multimodal_embeddings(
910
input_ids,
911
inputs_embeds,
948
- select_patch_features(multimodal_embeddings),
912
+ multimodal_embeddings,
913
self.img_context_token_id,
914
)
915
return inputs_embeds
vllm/model_executor/models/vision.py
+1
-76
@@ -1,8 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from abc import ABC, abstractmethod
4
-from collections.abc import Sequence
5
-from typing import Final, Generic, Optional, Protocol, TypeVar, Union, cast
4
+from typing import Final, Generic, Optional, Protocol, TypeVar, Union
5
6
import torch
7
from transformers import PretrainedConfig
@@ -10,12 +9,9 @@ from transformers import PretrainedConfig
9
import vllm.envs as envs
10
from vllm.attention.selector import (backend_name_to_enum,
11
get_global_forced_attn_backend)
13
-from vllm.jsontree import JSONTree, json_map_leaves
12
from vllm.logger import init_logger
13
from vllm.platforms import _Backend, current_platform
14
17
-from .interfaces import MultiModalEmbeddings
18
-
15
logger = init_logger(__name__)
16
17
_C = TypeVar("_C", bound=PretrainedConfig)
@@ -155,74 +151,3 @@ def resolve_visual_encoder_outputs(
151
if post_layer_norm is not None and uses_last_layer:
152
hs_pool[-1] = post_layer_norm(encoder_outputs)
153
return torch.cat(hs_pool, dim=-1)
158
-
159
-
160
-def scatter_patch_features(
161
- patches: Union[torch.Tensor, Sequence[torch.Tensor]],
162
- embed_is_patch: Union[torch.Tensor, Sequence[torch.Tensor]],
163
-) -> tuple[torch.Tensor, ...]:
164
- """
165
- Scatter the patch features into a contiguous tensor that corresponds
166
- to the embedding tokens defined by the multimodal processor.
167
-
168
- The rest of the values in the tensor are set to NaN so that they
169
- can be filtered out by :func`select_patch_features`.
170
-
171
- Args:
172
- patches: The patch features for each image.
173
- Shape: `(num_images, <patch_dims>, feature_depth)`
174
- embed_is_patch: A boolean mask indicating which image embeddings
175
- correspond to patch tokens for each image.
176
- Shape: `(num_images, num_embeds)`
177
-
178
- Note:
179
- The original code only considers patch tokens as feature
180
- tokens, but our processor considers all image-related tokens
181
- as feature tokens because the feature tokens need to be
182
- consecutive in `input_ids`.
183
-
184
- Example:
185
- A simplified example for one image:
186
-
187
- .. code-block::
188
-
189
- Embedding tokens (from HF processor):
190
- [<start> <patch> <patch> <col> <patch> <patch> <col> <end> ]
191
-
192
- embed_is_patch (from HF processor):
193
- [ False True True False True True False False ]
194
-
195
- Encoder outputs (from model):
196
- [ p1 p2 p3 p4 ]
197
-
198
- The resulting embedding tensor is:
199
- [ nan p1 p2 nan p3 p4 nan nan ]
200
- """
201
- if len(patches) != len(embed_is_patch):
202
- raise ValueError(f"Inconsistent num_images: {len(patches)=} vs. "
203
- f"{len(embed_is_patch)=}")
204
-
205
- def get_embed_one(patches_one: torch.Tensor, e_is_patch: torch.Tensor):
206
- embed_one = patches_one.new_full(
207
- (e_is_patch.shape[0], patches_one.shape[-1]),
208
- fill_value=torch.nan,
209
- )
210
- embed_one[e_is_patch] = patches_one
211
- return embed_one
212
-
213
- return tuple(
214
- get_embed_one(patches_one, e_is_patch)
215
- for patches_one, e_is_patch in zip(patches, embed_is_patch))
216
-
217
-
218
-def select_patch_features(
219
- multimodal_embeddings: MultiModalEmbeddings) -> MultiModalEmbeddings:
220
- """
221
- Given the outputs of :func:`scatter_patch_features`, return only
222
- the values that correspond to patch features.
223
- """
224
- selected_features = json_map_leaves(
225
- lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),
226
- cast(JSONTree[torch.Tensor], multimodal_embeddings),
227
- )
228
- return cast(MultiModalEmbeddings, selected_features)
vllm/multimodal/base.py
+2
-2
@@ -385,8 +385,8 @@ class MultiModalPlaceholderMap:
385
for placeholder_dict, mm_item in zip(multi_modal_placeholders,
386
multi_modal_items):
387
placeholder = range(
388
- placeholder_dict["offset"],
389
- placeholder_dict["offset"] + placeholder_dict["length"],
388
+ placeholder_dict.offset,
389
+ placeholder_dict.offset + placeholder_dict.length,
390
)
391
intersection = range(
392
max(positions.start, placeholder.start),
vllm/multimodal/inputs.py
+29
-3
@@ -109,7 +109,8 @@ The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
109
"""
110
111
112
-class PlaceholderRange(TypedDict):
112
+@dataclass(frozen=True)
113
+class PlaceholderRange:
114
"""
115
Placeholder location information for multi-modal data.
116
@@ -121,8 +122,8 @@ class PlaceholderRange(TypedDict):
122
123
.. code-block::
124
124
- A: { "offset": 0, "length": 4 }
125
- B: { "offset": 5, "length": 4 }
125
+ A: PlaceholderRange(offset=0, length=4)
126
+ B: PlaceholderRange(offset=5, length=4)
127
"""
128
129
offset: int
@@ -131,6 +132,31 @@ class PlaceholderRange(TypedDict):
132
length: int
133
"""The length of the placeholder."""
134
135
+ is_embed: Optional[torch.Tensor] = None
136
+ """
137
+ A boolean mask of shape `(length,)` indicating which positions
138
+ between `offset` and `offset + length` to assign embeddings to.
139
+ """
140
+
141
+ def get_num_embeds(self) -> int:
142
+ if self.is_embed is None:
143
+ return self.length
144
+
145
+ return int(self.is_embed.sum().item())
146
+
147
+ def __eq__(self, other: object) -> bool:
148
+ if not isinstance(other, self.__class__):
149
+ return False
150
+ if not (self.offset, self.length) == (other.offset, other.length):
151
+ return False
152
+
153
+ if self.is_embed is None:
154
+ return other.is_embed is None
155
+ if other.is_embed is None:
156
+ return self.is_embed is None
157
+
158
+ return nested_tensors_equal(self.is_embed, other.is_embed)
159
+
160
161
NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
162
tuple[torch.Tensor, ...]]
vllm/multimodal/processing.py
+52
-25
@@ -108,16 +108,46 @@ class PromptUpdateDetails(Generic[_S]):
108
full: _S
109
"""The full content."""
110
111
- features: _S
111
+ is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None
112
"""
113
- The part of the content that corresponds to feature placeholders;
114
- this will be replaced by the output of the vision encoder during model
115
- inference.
113
+ Given :attr:`full`, return a boolean mask of shape `(len(full),)`
114
+ indicating which positions of `full` to assign embeddings to.
115
+
116
+ `None` (default) means to assign embeddings to all positions of `full`.
117
+
118
+ The embeddings are obtained by calling
119
+ :class:`SupportsMultiModal.get_multimodal_embeddings`.
120
"""
121
122
@staticmethod
123
def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
120
- return PromptUpdateDetails(full=seq, features=seq)
124
+ return PromptUpdateDetails(full=seq)
125
+
126
+ @staticmethod
127
+ def select_text(
128
+ seq: _S,
129
+ embed_text: str,
130
+ ) -> "PromptUpdateDetails[_S]":
131
+
132
+ def is_embed(full: "_BoundPromptSequence") -> torch.Tensor:
133
+ embed_token_ids = encode_tokens(full.tokenizer, embed_text)
134
+
135
+ return torch.isin(
136
+ torch.tensor(full.token_ids),
137
+ torch.tensor(embed_token_ids),
138
+ )
139
+
140
+ return PromptUpdateDetails(full=seq, is_embed=is_embed)
141
+
142
+ @staticmethod
143
+ def select_token_id(
144
+ seq: _S,
145
+ embed_token_id: int,
146
+ ) -> "PromptUpdateDetails[_S]":
147
+ return PromptUpdateDetails(
148
+ full=seq,
149
+ is_embed=lambda f: torch.tensor(f.token_ids) == embed_token_id,
150
+ )
151
152
153
PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
@@ -406,7 +436,7 @@ class _BoundPromptSequence:
436
@dataclass
437
class _BoundPromptContent:
438
full: _BoundPromptSequence
409
- features: _BoundPromptSequence
439
+ is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]]
440
441
442
@dataclass
@@ -466,10 +496,8 @@ class BoundPromptUpdate:
496
497
bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
498
content.full)
469
- bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
470
- content.features)
499
bound_content = _BoundPromptContent(full=bound_full,
472
- features=bound_features)
500
+ is_embed=content.is_embed)
501
502
if cache_key is not None:
503
self._content_cache[cache_key] = bound_content
@@ -605,15 +633,19 @@ class PlaceholderFeaturesInfo:
633
item_idx: int
634
start_idx: int
635
tokens: list[int]
636
+ is_embed: Optional[torch.Tensor]
637
638
@property
639
def length(self) -> int:
640
return len(self.tokens)
641
642
def to_range(self) -> PlaceholderRange:
643
+ # TODO: Is it worth it to optimize this by stripping the
644
+ # leading and ending positions where `is_embed=False`?
645
return PlaceholderRange(
646
offset=self.start_idx,
647
length=self.length,
648
+ is_embed=self.is_embed,
649
)
650
651
@@ -806,22 +838,17 @@ def _iter_placeholders(
838
continue
839
840
if prompt[start_idx:end_idx_full] == content_tokens_full:
809
- content_tokens_feat = content.features.token_ids
810
-
811
- try:
812
- match = next(
813
- iter_token_matches(content_tokens_full,
814
- content_tokens_feat))
815
- yield PlaceholderFeaturesInfo(
816
- modality=modality,
817
- item_idx=item_idx,
818
- start_idx=start_idx + match.start_idx,
819
- tokens=content_tokens_feat,
820
- )
821
- except StopIteration:
822
- raise AssertionError(
823
- f"{content_tokens_feat=} should be a "
824
- f"subsequence of {content_tokens_full=}") from None
841
+ content_is_embed = content.is_embed
842
+ if content_is_embed is not None:
843
+ content_is_embed = content_is_embed(content.full)
844
+
845
+ yield PlaceholderFeaturesInfo(
846
+ modality=modality,
847
+ item_idx=item_idx,
848
+ start_idx=start_idx,
849
+ tokens=content_tokens_full,
850
+ is_embed=content_is_embed,
851
+ )
852
853
# Exclude overlapping matches
854
start_idx = end_idx_full
vllm/multimodal/profiling.py
+1
-1
@@ -180,7 +180,7 @@ class MultiModalProfiler(Generic[_I]):
180
placeholders_by_modality = mm_inputs["mm_placeholders"]
181
182
total_placeholders_by_modality = {
183
- modality: sum(item["length"] for item in placeholders)
183
+ modality: sum(item.get_num_embeds() for item in placeholders)
184
for modality, placeholders in placeholders_by_modality.items()
185
}
186
expected_placeholders_by_modality = {
vllm/multimodal/utils.py
+1
-1
@@ -340,7 +340,7 @@ def merge_and_sort_multimodal_metadata(
340
all_items.append((modality, placeholder, hash_value))
341
342
# Sort all items by offset
343
- all_items.sort(key=lambda x: x[1]['offset'])
343
+ all_items.sort(key=lambda x: x[1].offset)
344
345
# Split into separate lists
346
sorted_modalities = [item[0] for item in all_items]
vllm/v1/core/kv_cache_utils.py
+3
-4
@@ -310,8 +310,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
310
# Note that we assume mm_positions is sorted by offset.
311
# We do not need to check all mm inputs if the start token index is out of
312
# range. This usually happens in the late prefill phase and decoding phase.
313
- if mm_positions[-1]["offset"] + mm_positions[-1][
314
- "length"] < start_token_idx:
313
+ if mm_positions[-1].offset + mm_positions[-1].length < start_token_idx:
314
return extra_keys, start_mm_idx
315
316
# Support start_mm_idx == -1 to indicate the last mm input.
@@ -322,8 +321,8 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
321
curr_mm_idx = start_mm_idx
322
while mm_positions and curr_mm_idx < len(mm_positions):
323
assert mm_hashes[curr_mm_idx] is not None
325
- offset = mm_positions[curr_mm_idx]["offset"]
326
- length = mm_positions[curr_mm_idx]["length"]
324
+ offset = mm_positions[curr_mm_idx].offset
325
+ length = mm_positions[curr_mm_idx].length
326
if end_token_idx > offset:
327
if start_token_idx > offset + length:
328
# This block has passed the current mm input.
vllm/v1/core/sched/scheduler.py
+4
-4
@@ -505,8 +505,8 @@ class Scheduler(SchedulerInterface):
505
assert mm_positions is not None
506
assert len(mm_positions) > 0
507
for i, pos_info in enumerate(mm_positions):
508
- start_pos = pos_info["offset"]
509
- num_encoder_tokens = pos_info["length"]
508
+ start_pos = pos_info.offset
509
+ num_encoder_tokens = pos_info.length
510
511
# The encoder output is needed if the two ranges overlap:
512
# [num_computed_tokens, num_computed_tokens + num_new_tokens) and
@@ -596,8 +596,8 @@ class Scheduler(SchedulerInterface):
596
if cached_encoder_input_ids:
597
for input_id in list(cached_encoder_input_ids):
598
mm_positions = request.mm_positions[input_id]
599
- start_pos = mm_positions["offset"]
600
- num_tokens = mm_positions["length"]
599
+ start_pos = mm_positions.offset
600
+ num_tokens = mm_positions.length
601
if start_pos + num_tokens <= request.num_computed_tokens:
602
# The encoder output is already processed and stored
603
# in the decoder's KV cache.
vllm/v1/request.py
+1
-1
@@ -121,7 +121,7 @@ class Request:
121
122
def get_num_encoder_tokens(self, input_id: int) -> int:
123
assert input_id < len(self.mm_positions)
124
- num_tokens = self.mm_positions[input_id]["length"]
124
+ num_tokens = self.mm_positions[input_id].length
125
return num_tokens
126
127
@property
vllm/v1/worker/gpu_model_runner.py
+40
-20
@@ -19,7 +19,8 @@ from vllm.logger import init_logger
19
from vllm.model_executor.layers.fused_moe import FusedMoE
20
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
21
from vllm.model_executor.model_loader import get_model
22
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
22
+from vllm.multimodal import MULTIMODAL_REGISTRY
23
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
24
from vllm.multimodal.utils import group_mm_inputs_by_modality
25
from vllm.sampling_params import SamplingType
26
from vllm.sequence import IntermediateTensors
@@ -43,7 +44,8 @@ from vllm.v1.utils import bind_kv_cache
44
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
45
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
46
46
-from .utils import sanity_check_mm_encoder_outputs
47
+from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,
48
+ scatter_mm_placeholders)
49
50
if TYPE_CHECKING:
51
import xgrammar as xgr
@@ -829,19 +831,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
831
)
832
return metadata
833
832
- def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
834
+ def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
835
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
836
if not scheduled_encoder_inputs:
837
return
838
839
# Batch the multi-modal inputs.
838
- mm_inputs: list[MultiModalKwargs] = []
839
- req_input_ids: list[tuple[str, int]] = []
840
+ mm_inputs = list[MultiModalKwargs]()
841
+ req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
842
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
843
req_state = self.requests[req_id]
842
- for input_id in encoder_input_ids:
844
+ for input_id, pos_info in zip(
845
+ encoder_input_ids,
846
+ req_state.mm_positions,
847
+ ):
848
mm_inputs.append(req_state.mm_inputs[input_id])
844
- req_input_ids.append((req_id, input_id))
849
+ req_ids_pos.append((req_id, input_id, pos_info))
850
851
# Batch mm inputs as much as we can: if a request in the batch has
852
# multiple modalities or a different modality than the previous one,
@@ -877,16 +882,23 @@ class GPUModelRunner(LoRAModelRunnerMixin):
882
encoder_outputs.append(output)
883
884
# Cache the encoder outputs.
880
- for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
885
+ for (req_id, input_id, pos_info), output in zip(
886
+ req_ids_pos,
887
+ encoder_outputs,
888
+ ):
889
if req_id not in self.encoder_cache:
890
self.encoder_cache[req_id] = {}
883
- self.encoder_cache[req_id][input_id] = output
891
885
- def _gather_encoder_outputs(
892
+ self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(
893
+ output,
894
+ is_embed=pos_info.is_embed,
895
+ )
896
+
897
+ def _gather_mm_embeddings(
898
self,
899
scheduler_output: "SchedulerOutput",
900
) -> list[torch.Tensor]:
889
- encoder_outputs: list[torch.Tensor] = []
901
+ mm_embeds: list[torch.Tensor] = []
902
for req_id in self.input_batch.req_ids:
903
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
904
req_id]
@@ -894,8 +906,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
906
num_computed_tokens = req_state.num_computed_tokens
907
mm_positions = req_state.mm_positions
908
for i, pos_info in enumerate(mm_positions):
897
- start_pos = pos_info["offset"]
898
- num_encoder_tokens = pos_info["length"]
909
+ start_pos = pos_info.offset
910
+ num_encoder_tokens = pos_info.length
911
912
# The encoder output is needed if the two ranges overlap:
913
# [num_computed_tokens,
@@ -917,8 +929,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
929
assert req_id in self.encoder_cache
930
assert i in self.encoder_cache[req_id]
931
encoder_output = self.encoder_cache[req_id][i]
920
- encoder_outputs.append(encoder_output[start_idx:end_idx])
921
- return encoder_outputs
932
+
933
+ if (is_embed := pos_info.is_embed) is not None:
934
+ is_embed = is_embed[start_idx:end_idx]
935
+
936
+ mm_embeds_item = gather_mm_placeholders(
937
+ encoder_output[start_idx:end_idx],
938
+ is_embed=is_embed,
939
+ )
940
+ mm_embeds.append(mm_embeds_item)
941
+ return mm_embeds
942
943
def get_model(self) -> nn.Module:
944
return self.model
@@ -983,10 +1003,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1003
1004
if self.is_multimodal_model:
1005
# Run the multimodal encoder if any.
986
- self._execute_encoder(scheduler_output)
987
- encoder_outputs = self._gather_encoder_outputs(scheduler_output)
1006
+ self._execute_mm_encoder(scheduler_output)
1007
+ mm_embeds = self._gather_mm_embeddings(scheduler_output)
1008
else:
989
- encoder_outputs = []
1009
+ mm_embeds = []
1010
1011
# Prepare the decoder inputs.
1012
attn_metadata, logits_indices, spec_decode_metadata = (
@@ -1008,9 +1028,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1028
# embeddings), we always use embeddings (rather than token ids)
1029
# as input to the multimodal model, even when the input is text.
1030
input_ids = self.input_ids[:num_scheduled_tokens]
1011
- if encoder_outputs:
1031
+ if mm_embeds:
1032
inputs_embeds = self.model.get_input_embeddings(
1013
- input_ids, encoder_outputs)
1033
+ input_ids, mm_embeds)
1034
else:
1035
inputs_embeds = self.model.get_input_embeddings(input_ids)
1036
# TODO(woosuk): Avoid the copy. Optimize.
vllm/v1/worker/tpu_model_runner.py
+65
-20
@@ -19,7 +19,8 @@ from vllm.config import VllmConfig
19
from vllm.forward_context import set_forward_context
20
from vllm.logger import init_logger
21
from vllm.model_executor.model_loader import get_model
22
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
22
+from vllm.multimodal import MULTIMODAL_REGISTRY
23
+from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
24
from vllm.multimodal.utils import group_mm_inputs_by_modality
25
from vllm.sampling_params import SamplingType
26
from vllm.sequence import IntermediateTensors
@@ -36,7 +37,8 @@ from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler
37
from vllm.v1.utils import bind_kv_cache
38
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
39
39
-from .utils import sanity_check_mm_encoder_outputs
40
+from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,
41
+ scatter_mm_placeholders)
42
43
if TYPE_CHECKING:
44
from vllm.v1.core.sched.output import SchedulerOutput
@@ -507,19 +509,47 @@ class TPUModelRunner:
509
logits_indices = logits_indices.to(self.device)
510
return attn_metadata, logits_indices
511
510
- def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
512
+ def _scatter_placeholders(
513
+ self,
514
+ embeds: torch.Tensor,
515
+ is_embed: Optional[torch.Tensor],
516
+ ) -> torch.Tensor:
517
+ if is_embed is None:
518
+ return embeds
519
+
520
+ placeholders = embeds.new_full(
521
+ (is_embed.shape[0], embeds.shape[-1]),
522
+ fill_value=torch.nan,
523
+ )
524
+ placeholders[is_embed] = embeds
525
+ return placeholders
526
+
527
+ def _gather_placeholders(
528
+ self,
529
+ placeholders: torch.Tensor,
530
+ is_embed: Optional[torch.Tensor],
531
+ ) -> torch.Tensor:
532
+ if is_embed is None:
533
+ return placeholders
534
+
535
+ return placeholders[is_embed]
536
+
537
+ def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
538
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
539
if not scheduled_encoder_inputs:
540
return
541
542
# Batch the multi-modal inputs.
516
- mm_inputs: list[MultiModalKwargs] = []
517
- req_input_ids: list[tuple[str, int]] = []
543
+ mm_inputs = list[MultiModalKwargs]()
544
+ req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
545
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
546
req_state = self.requests[req_id]
520
- for input_id in encoder_input_ids:
547
+ for input_id, pos_info in zip(
548
+ encoder_input_ids,
549
+ req_state.mm_positions,
550
+ ):
551
mm_inputs.append(req_state.mm_inputs[input_id])
522
- req_input_ids.append((req_id, input_id))
552
+ req_ids_pos.append((req_id, input_id, pos_info))
553
554
# Batch mm inputs as much as we can: if a request in the batch has
555
# multiple modalities or a different modality than the previous one,
@@ -555,16 +585,23 @@ class TPUModelRunner:
585
encoder_outputs.append(output)
586
587
# Cache the encoder outputs.
558
- for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
588
+ for (req_id, input_id, pos_info), output in zip(
589
+ req_ids_pos,
590
+ encoder_outputs,
591
+ ):
592
if req_id not in self.encoder_cache:
593
self.encoder_cache[req_id] = {}
561
- self.encoder_cache[req_id][input_id] = output
594
563
- def _gather_encoder_outputs(
595
+ self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(
596
+ output,
597
+ is_embed=pos_info.is_embed,
598
+ )
599
+
600
+ def _gather_mm_embeddings(
601
self,
602
scheduler_output: "SchedulerOutput",
603
) -> list[torch.Tensor]:
567
- encoder_outputs: list[torch.Tensor] = []
604
+ mm_embeds: list[torch.Tensor] = []
605
for req_id in self.input_batch.req_ids:
606
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
607
req_id]
@@ -572,8 +609,8 @@ class TPUModelRunner:
609
num_computed_tokens = req_state.num_computed_tokens
610
mm_positions = req_state.mm_positions
611
for i, pos_info in enumerate(mm_positions):
575
- start_pos = pos_info["offset"]
576
- num_encoder_tokens = pos_info["length"]
612
+ start_pos = pos_info.offset
613
+ num_encoder_tokens = pos_info.length
614
615
# The encoder output is needed if the two ranges overlap:
616
# [num_computed_tokens,
@@ -595,8 +632,16 @@ class TPUModelRunner:
632
assert req_id in self.encoder_cache
633
assert i in self.encoder_cache[req_id]
634
encoder_output = self.encoder_cache[req_id][i]
598
- encoder_outputs.append(encoder_output[start_idx:end_idx])
599
- return encoder_outputs
635
+
636
+ if (is_embed := pos_info.is_embed) is not None:
637
+ is_embed = is_embed[start_idx:end_idx]
638
+
639
+ mm_embeds_item = gather_mm_placeholders(
640
+ encoder_output[start_idx:end_idx],
641
+ is_embed=is_embed,
642
+ )
643
+ mm_embeds.append(mm_embeds_item)
644
+ return mm_embeds
645
646
@torch.no_grad()
647
def execute_model(
@@ -612,10 +657,10 @@ class TPUModelRunner:
657
658
if self.is_multimodal_model:
659
# Run the multimodal encoder if any.
615
- self._execute_encoder(scheduler_output)
616
- encoder_outputs = self._gather_encoder_outputs(scheduler_output)
660
+ self._execute_mm_encoder(scheduler_output)
661
+ mm_embeds = self._gather_mm_embeddings(scheduler_output)
662
else:
618
- encoder_outputs = []
663
+ mm_embeds = []
664
665
# Prepare inputs
666
attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
@@ -623,9 +668,9 @@ class TPUModelRunner:
668
# NOTE(woosuk): To unify token ids and soft tokens (vision
669
# embeddings), we always use embeddings (rather than token ids)
670
# as input to the multimodal model, even when the input is text.
626
- if encoder_outputs:
671
+ if mm_embeds:
672
inputs_embeds = self.model.get_input_embeddings(
628
- self.input_ids, encoder_outputs)
673
+ self.input_ids, mm_embeds)
674
else:
675
inputs_embeds = self.model.get_input_embeddings(self.input_ids)
676
input_ids = None
vllm/v1/worker/utils.py
+45
@@ -1,4 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+from typing import Optional
3
+
4
import torch
5
6
@@ -27,3 +29,46 @@ def sanity_check_mm_encoder_outputs(
29
f"but got tensors with shapes {[e.shape for e in mm_embeddings]} "
30
"instead. This is most likely due to incorrect implementation "
31
"of the model's `get_multimodal_embeddings` method.")
32
+
33
+
34
+def scatter_mm_placeholders(
35
+ embeds: torch.Tensor,
36
+ is_embed: Optional[torch.Tensor],
37
+) -> torch.Tensor:
38
+ """
39
+ Scatter the multimodal embeddings into a contiguous tensor that represents
40
+ the placeholder tokens.
41
+
42
+ :class:`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
43
+
44
+ Args:
45
+ embeds: The multimodal embeddings.
46
+ Shape: `(num_embeds, embed_dim)`
47
+ is_embed: A boolean mask indicating which positions in the placeholder
48
+ tokens need to be filled with multimodal embeddings.
49
+ Shape: `(num_placeholders, num_embeds)`
50
+ """
51
+ if is_embed is None:
52
+ return embeds
53
+
54
+ placeholders = embeds.new_full(
55
+ (is_embed.shape[0], embeds.shape[-1]),
56
+ fill_value=torch.nan,
57
+ )
58
+ placeholders[is_embed] = embeds
59
+ return placeholders
60
+
61
+
62
+def gather_mm_placeholders(
63
+ placeholders: torch.Tensor,
64
+ is_embed: Optional[torch.Tensor],
65
+) -> torch.Tensor:
66
+ """
67
+ Reconstructs the embeddings from the placeholder tokens.
68
+
69
+ This is the operation of :func:`scatter_mm_placeholders`.
70
+ """
71
+ if is_embed is None:
72
+ return placeholders
73
+
74
+ return placeholders[is_embed]