@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af51d80fa1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

Revert "[V1] Scatter and gather placeholders in the model runner" (#16075)

Roger Wang committed Apr 4, 2025 at 14:50 UTC af51d80fa14ca8e01c6be36232170683f3e47f09
42 files changed +943 -497
docs/source/contributing/model/multimodal.md
+8 -8
@@ -860,8 +860,8 @@ prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch(
860 )
861 ```
862
863 -To assign the vision embeddings to only the image tokens, instead of a string
864 -you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`:
863 +To accommodate this, instead of a string you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`
864 +with different `full` and `feature` attributes:
865
866 ```python
867 hf_config = self.info.get_hf_config()
@@ -879,9 +879,9 @@ def get_replacement_fuyu(item_idx: int):
879 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
880 [_NEWLINE_TOKEN_ID]) * nrows
881
882 - return PromptUpdateDetails.select_token_id(
883 - image_tokens + [bos_token_id],
884 - embed_token_id=_IMAGE_TOKEN_ID,
882 + return PromptUpdateDetails(
883 + full=image_tokens + [bos_token_id],
884 + features=image_tokens,
885 )
886 ```
887
@@ -914,9 +914,9 @@ def _get_prompt_updates(
914 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
915 [_NEWLINE_TOKEN_ID]) * nrows
916
917 - return PromptUpdateDetails.select_token_id(
918 - image_tokens + [bos_token_id],
919 - embed_token_id=_IMAGE_TOKEN_ID,
917 + return PromptUpdateDetails(
918 + full=image_tokens + [bos_token_id],
919 + features=image_tokens,
920 )
921
922 return [
docs/source/models/supported_models.md
+3
@@ -989,6 +989,9 @@ See [this page](#generative-models) for more information on how to use generativ
989 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.
990
991 :::{important}
992 +To use Gemma3 series models, you have to install Hugging Face Transformers library from source via
993 +`pip install git+https://github.com/huggingface/transformers`.
994 +
995 Pan-and-scan image pre-processing is currently supported on V0 (but not V1).
996 You can enable it by passing `--mm-processor-kwargs '{"do_pan_and_scan": True}'`.
997 :::
examples/offline_inference/audio_language.py
+1 -1
@@ -47,7 +47,7 @@ def run_minicpmo(question: str, audio_count: int) -> ModelRequestData:
47 model=model_name,
48 trust_remote_code=True,
49 max_model_len=4096,
50 - max_num_seqs=2,
50 + max_num_seqs=5,
51 limit_mm_per_prompt={"audio": audio_count},
52 )
53
tests/models/decoder_only/audio_language/test_ultravox.py
+1 -4
@@ -55,10 +55,7 @@ def server(request, audio_assets):
55 for key, value in request.param.items()
56 ]
57
58 - with RemoteOpenAIServer(MODEL_NAME,
59 - args,
60 - env_dict={"VLLM_AUDIO_FETCH_TIMEOUT":
61 - "30"}) as remote_server:
58 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
59 yield remote_server
60
61
tests/models/decoder_only/vision_language/test_models.py
+1 -1
@@ -167,7 +167,7 @@ VLM_TEST_SETTINGS = {
167 "cherry_blossom": "<image>What is the season?", # noqa: E501
168 }),
169 multi_image_prompt="<image><image>Describe the two images in detail.", # noqa: E501
170 - max_model_len=4096,
170 + max_model_len=8192,
171 max_num_seqs=2,
172 auto_cls=AutoModelForImageTextToText,
173 vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}}
tests/models/decoder_only/vision_language/test_pixtral.py
+16 -10
@@ -176,8 +176,6 @@ def test_chat(
176 model,
177 dtype=dtype,
178 tokenizer_mode="mistral",
179 - load_format="mistral",
180 - config_format="mistral",
179 max_model_len=max_model_len,
180 limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
181 ) as vllm_model:
@@ -200,14 +198,22 @@ def test_chat(
198
199
200 @large_gpu_test(min_gb=48)
203 -@pytest.mark.parametrize("prompt,expected_ranges",
204 - [(_create_engine_inputs_hf(IMG_URLS[:1]),
205 - [PlaceholderRange(offset=11, length=494)]),
206 - (_create_engine_inputs_hf(IMG_URLS[1:4]), [
207 - PlaceholderRange(offset=11, length=266),
208 - PlaceholderRange(offset=277, length=1056),
209 - PlaceholderRange(offset=1333, length=418)
210 - ])])
201 +@pytest.mark.parametrize(
202 + "prompt,expected_ranges",
203 + [(_create_engine_inputs_hf(IMG_URLS[:1]), [{
204 + "offset": 11,
205 + "length": 494
206 + }]),
207 + (_create_engine_inputs_hf(IMG_URLS[1:4]), [{
208 + "offset": 11,
209 + "length": 266
210 + }, {
211 + "offset": 277,
212 + "length": 1056
213 + }, {
214 + "offset": 1333,
215 + "length": 418
216 + }])])
217 def test_multi_modal_placeholders(vllm_runner, prompt,
218 expected_ranges: list[PlaceholderRange],
219 monkeypatch) -> None:
tests/models/multimodal/processing/test_llava_next.py
+2 -2
@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(
92 first_placeholder = image_placeholders[0]
93
94 # NOTE: There is a BOS token
95 - assert first_placeholder.offset == 1
96 - assert first_placeholder.length == (
95 + assert first_placeholder["offset"] == 1
96 + assert first_placeholder["length"] == (
97 len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
98
99 except Exception as exc:
tests/models/multimodal/processing/test_llava_onevision.py
+2 -2
@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(
92
93 first_placeholder = image_placeholders[0]
94
95 - assert first_placeholder.offset == 0
96 - assert first_placeholder.length == len(
95 + assert first_placeholder["offset"] == 0
96 + assert first_placeholder["length"] == len(
97 processed_inputs["prompt_token_ids"]) // num_imgs
98 except Exception as exc:
99 failed_size_excs.append((image_size, exc))
tests/models/registry.py
+1 -3
@@ -277,9 +277,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
277 trust_remote_code=True,
278 hf_overrides={"architectures": ["GLM4VForCausalLM"]}), # noqa: E501
279 "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m",
280 - extras={"2b": "h2oai/h2ovl-mississippi-2b"}, # noqa: E501
281 - max_transformers_version="4.48", # noqa: E501
282 - transformers_version_reason="HF model is not compatible."), # noqa: E501
280 + extras={"2b": "h2oai/h2ovl-mississippi-2b"}), # noqa: E501
281 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
282 extras={"2B": "OpenGVLab/InternVL2-2B"}, # noqa: E501
283 trust_remote_code=True),
tests/multimodal/test_processing.py
-9
@@ -785,7 +785,6 @@ def test_find_update_tokens(
785 item_idx=0,
786 start_idx=6,
787 tokens=[32000, 32000],
788 - is_embed=None,
788 ),
789 ],
790 "pattern_4": [
@@ -794,7 +793,6 @@ def test_find_update_tokens(
793 item_idx=0,
794 start_idx=3,
795 tokens=[32000],
797 - is_embed=None,
796 ),
797 ],
798 }
@@ -809,14 +807,12 @@ def test_find_update_tokens(
807 item_idx=0,
808 start_idx=1,
809 tokens=[32000, 32000],
812 - is_embed=None,
810 ),
811 PlaceholderFeaturesInfo(
812 modality="pattern_1",
813 item_idx=1,
814 start_idx=5,
815 tokens=[32000, 32000],
819 - is_embed=None,
816 ),
817 ],
818 "pattern_3": [
@@ -825,7 +821,6 @@ def test_find_update_tokens(
821 item_idx=0,
822 start_idx=7,
823 tokens=[1550, 918, 1550],
828 - is_embed=None,
824 ),
825 ],
826 # No match for pattern_4 as it has lower priority than pattern_1
@@ -840,14 +835,12 @@ def test_find_update_tokens(
835 item_idx=0,
836 start_idx=1,
837 tokens=[32000, 32000],
843 - is_embed=None,
838 ),
839 PlaceholderFeaturesInfo(
840 modality="pattern_1",
841 item_idx=1,
842 start_idx=3,
843 tokens=[32000, 32000],
850 - is_embed=None,
844 ),
845 ],
846 "pattern_4": [
@@ -856,7 +849,6 @@ def test_find_update_tokens(
849 item_idx=0,
850 start_idx=5,
851 tokens=[32000],
859 - is_embed=None,
852 ),
853 ],
854 "pattern_3": [
@@ -865,7 +857,6 @@ def test_find_update_tokens(
857 item_idx=0,
858 start_idx=6,
859 tokens=[1550, 918, 1550],
868 - is_embed=None,
860 ),
861 ],
862 }
tests/v1/core/test_kv_cache_utils.py
+29 -17
@@ -3,7 +3,7 @@
3 import pytest
4 import torch
5
6 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
6 +from vllm.multimodal.inputs import MultiModalKwargs
7 from vllm.sampling_params import SamplingParams
8 from vllm.utils import sha256
9 # disable yapf here as it formats differently than isort such that both fail
@@ -158,10 +158,13 @@ def test_generate_block_hash_extra_keys():
158 request = make_request(
159 request_id=0,
160 prompt_token_ids=[_ for _ in range(20)],
161 - mm_positions=[
162 - PlaceholderRange(offset=0, length=5),
163 - PlaceholderRange(offset=10, length=5),
164 - ],
161 + mm_positions=[{
162 + "offset": 0,
163 + "length": 5
164 + }, {
165 + "offset": 10,
166 + "length": 5
167 + }],
168 mm_hashes=["hash1", "hash2"],
169 )
170
@@ -219,10 +222,13 @@ def test_hash_request_tokens(hash_fn):
222 request = make_request(
223 request_id=0,
224 prompt_token_ids=[_ for _ in range(6)],
222 - mm_positions=[
223 - PlaceholderRange(offset=0, length=3),
224 - PlaceholderRange(offset=3, length=3),
225 - ],
225 + mm_positions=[{
226 + "offset": 0,
227 + "length": 3
228 + }, {
229 + "offset": 3,
230 + "length": 3
231 + }],
232 mm_hashes=["hash1", "hash2"],
233 )
234
@@ -247,19 +253,25 @@ def test_hash_tokens_different_mm_input(hash_fn):
253 request1 = make_request(
254 request_id=0,
255 prompt_token_ids=[_ for _ in range(6)],
250 - mm_positions=[
251 - PlaceholderRange(offset=0, length=3),
252 - PlaceholderRange(offset=3, length=3),
253 - ],
256 + mm_positions=[{
257 + "offset": 0,
258 + "length": 3
259 + }, {
260 + "offset": 3,
261 + "length": 3
262 + }],
263 mm_hashes=["hash1", "hash2"],
264 )
265 request2 = make_request(
266 request_id=1,
267 prompt_token_ids=[_ for _ in range(6)],
259 - mm_positions=[
260 - PlaceholderRange(offset=0, length=3),
261 - PlaceholderRange(offset=3, length=3),
262 - ],
268 + mm_positions=[{
269 + "offset": 0,
270 + "length": 3
271 + }, {
272 + "offset": 3,
273 + "length": 3
274 + }],
275 mm_hashes=["hash3", "hash2"],
276 )
277 block_size = 3
vllm/model_executor/models/aya_vision.py
+50 -21
@@ -27,7 +27,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 BaseProcessingInfo,
28 MultiModalFieldConfig,
29 PromptReplacement, PromptUpdate,
30 - PromptUpdateDetails)
30 + encode_tokens)
31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 from vllm.sequence import IntermediateTensors
33
@@ -35,6 +35,7 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35 from .siglip import SiglipVisionModel
36 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
37 maybe_prefix, merge_multimodal_embeddings)
38 +from .vision import scatter_patch_features, select_patch_features
39
40
41 class AyaVisionImagePixelInputs(TypedDict):
@@ -50,6 +51,13 @@ class AyaVisionImagePixelInputs(TypedDict):
51 num_patches: torch.Tensor
52 """Shape: `(batch_size * num_images)`"""
53
54 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
55 + """
56 + A boolean mask indicating which image embeddings correspond to patch tokens.
57 +
58 + Shape: `(batch_size * num_images, num_embeds)`
59 + """
60 +
61
62 class AyaVisionMultiModalProjector(nn.Module):
63
@@ -127,20 +135,21 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
135 def get_max_image_tokens(self) -> int:
136 hf_processor = self.get_hf_processor()
137 image_processor = hf_processor.image_processor
130 -
138 image_size = self.get_image_size_with_most_features()
139 + tokenizer = hf_processor.tokenizer
140 num_patches = self.get_num_patches(
141 image_width=image_size.width,
142 image_height=image_size.height,
143 size=image_processor.size,
144 min_patches=image_processor.min_patches,
137 - max_patches=image_processor.max_patches,
145 + max_patches=image_processor.max_patches)
146 + image_string = hf_processor._prompt_split_image(num_patches)
147 + x = encode_tokens(
148 + tokenizer,
149 + image_string,
150 + add_special_tokens=False,
151 )
139 -
140 - img_patches_per_tile = (hf_processor.img_size //
141 - hf_processor.patch_size)**2
142 -
143 - return num_patches * img_patches_per_tile
152 + return len(x)
153
154 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
155 return {"image": None}
@@ -212,6 +221,7 @@ class AyaVisionMultiModalProcessor(
221 hf_processor = self.info.get_hf_processor(**mm_kwargs)
222 image_processor = hf_processor.image_processor
223
224 + hf_config = self.info.get_hf_config()
225 # HF processor pops the `num_patches` kwarg, which is needed by vLLM
226 if (images :=
227 mm_data.get("images")) is not None and '<image>' in prompt:
@@ -224,7 +234,6 @@ class AyaVisionMultiModalProcessor(
234 parsed_images.get_image_size(i)
235 for i in range(len(parsed_images))
236 ]
227 -
237 num_patches = [
238 self.info.get_num_patches(
239 image_width=image_size.width,
@@ -234,6 +243,20 @@ class AyaVisionMultiModalProcessor(
243 max_patches=image_processor.max_patches)
244 for image_size in image_sizes
245 ]
246 + image_tokens_list = [
247 + hf_processor._prompt_split_image(num_patch)
248 + for num_patch in num_patches
249 + ]
250 + tokenizer = self.info.get_tokenizer()
251 + image_token_ids = [
252 + tokenizer.encode(image_tokens, add_special_tokens=False)
253 + for image_tokens in image_tokens_list
254 + ]
255 + embed_is_patch = [
256 + torch.tensor(image_repl_tokens) == hf_config.image_token_index
257 + for image_repl_tokens in image_token_ids
258 + ]
259 + processed_outputs["embed_is_patch"] = embed_is_patch
260 processed_outputs["num_patches"] = torch.tensor(num_patches)
261
262 return processed_outputs
@@ -248,6 +271,7 @@ class AyaVisionMultiModalProcessor(
271 pixel_values=MultiModalFieldConfig.flat_from_sizes(
272 "image", num_patches),
273 num_patches=MultiModalFieldConfig.batched("image"),
274 + embed_is_patch=MultiModalFieldConfig.batched("image"),
275 image_embeds=MultiModalFieldConfig.batched("image"),
276 )
277
@@ -259,7 +283,6 @@ class AyaVisionMultiModalProcessor(
283 ) -> Sequence[PromptUpdate]:
284 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
285 image_token = hf_processor.image_token
262 - img_patch_token = hf_processor.img_patch_token
286 image_processor = hf_processor.image_processor
287
288 def get_replacement(item_idx: int):
@@ -271,11 +294,8 @@ class AyaVisionMultiModalProcessor(
294 image_height=image_size.height,
295 size=image_processor.size,
296 min_patches=image_processor.min_patches,
274 - max_patches=image_processor.max_patches,
275 - )
276 - repl = hf_processor._prompt_split_image(num_patches=num_patches)
277 -
278 - return PromptUpdateDetails.select_text(repl, img_patch_token)
297 + max_patches=image_processor.max_patches)
298 + return hf_processor._prompt_split_image(num_patches=num_patches)
299
300 return [
301 PromptReplacement(
@@ -404,6 +424,7 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
424 self, **kwargs: object) -> Optional[AyaVisionImagePixelInputs]:
425 pixel_values = kwargs.pop("pixel_values", None)
426 num_patches = kwargs.pop("num_patches", None)
427 + embed_is_patch = kwargs.pop("embed_is_patch", None)
428 image_embeds = kwargs.pop("image_embeds", None)
429 assert image_embeds is None, "Aya Vision does not support image_embeds."
430
@@ -415,13 +436,18 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
436 raise ValueError("Incorrect type of num_patches. "
437 f"Got type: {type(num_patches)}")
438
439 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
440 + raise ValueError("Incorrect type of embed_is_patch. "
441 + f"Got type: {type(embed_is_patch)}")
442 +
443 pixel_values = flatten_bn(pixel_values, concat=True)
444 num_patches = flatten_bn(num_patches, concat=True)
420 -
445 + embed_is_patch = flatten_bn(embed_is_patch)
446 return AyaVisionImagePixelInputs(
447 type="pixel_values",
448 pixel_values=self._validate_pixel_values(pixel_values),
449 num_patches=num_patches,
450 + embed_is_patch=embed_is_patch,
451 )
452
453 def get_multimodal_embeddings(
@@ -429,8 +455,11 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
455 image_input = self._parse_and_validate_image_input(**kwargs)
456 if image_input is None:
457 return None
432 -
433 - return self._process_image_input(image_input, **kwargs)
458 + image_features = self._process_image_input(image_input, **kwargs)
459 + return scatter_patch_features(
460 + image_features,
461 + image_input["embed_is_patch"],
462 + )
463
464 def get_input_embeddings(
465 self,
@@ -442,9 +471,9 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,
471 inputs_embeds = merge_multimodal_embeddings(
472 input_ids=input_ids,
473 inputs_embeds=inputs_embeds,
445 - multimodal_embeddings=multimodal_embeddings,
446 - placeholder_token_id=self.config.image_token_index,
447 - )
474 + multimodal_embeddings=select_patch_features(
475 + multimodal_embeddings),
476 + placeholder_token_id=self.config.image_token_index)
477
478 return inputs_embeds
479
vllm/model_executor/models/chameleon.py
+3 -3
@@ -162,9 +162,9 @@ class ChameleonMultiModalProcessor(
162 PromptReplacement(
163 modality="image",
164 target=[image_token_id],
165 - replacement=PromptUpdateDetails.select_token_id(
166 - [image_start_id] + image_tokens + [image_end_id],
167 - embed_token_id=image_token_id,
165 + replacement=PromptUpdateDetails(
166 + full=([image_start_id] + image_tokens + [image_end_id]),
167 + features=image_tokens,
168 ),
169 )
170 ]
vllm/model_executor/models/fuyu.py
+52 -33
@@ -18,7 +18,7 @@
18 """ PyTorch Fuyu model."""
19 import math
20 from collections.abc import Iterable, Mapping, Sequence
21 -from typing import Literal, Optional, Set, Tuple, TypedDict
21 +from typing import Literal, Optional, Set, Tuple, TypedDict, Union
22
23 import torch
24 import torch.nn as nn
@@ -43,6 +43,7 @@ from vllm.sequence import IntermediateTensors
43 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
44 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
45 merge_multimodal_embeddings)
46 +from .vision import scatter_patch_features, select_patch_features
47
48 # Cannot find the following 2 numbers from hf config.
49 _IMAGE_TOKEN_ID = 71011
@@ -65,6 +66,14 @@ class FuyuImagePatchInputs(TypedDict):
66 flattened just like `flat_data`.
67 """
68
69 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
70 + """
71 + A boolean mask indicating which image embeddings correspond
72 + to patch tokens.
73 +
74 + Shape: `(batch_size * num_images, num_embeds)`
75 + """
76 +
77
78 class FuyuProcessingInfo(BaseProcessingInfo):
79
@@ -85,7 +94,15 @@ class FuyuProcessingInfo(BaseProcessingInfo):
94 seq_len: int,
95 mm_counts: Mapping[str, int],
96 ) -> Mapping[str, int]:
88 - return {"image": self.get_max_image_tokens()}
97 + target_width, target_height = self.get_image_size_with_most_features()
98 +
99 + max_ncols, max_nrows = self.get_image_feature_grid_size(
100 + image_width=target_width,
101 + image_height=target_height,
102 + )
103 + max_image_tokens = (max_ncols + 1) * max_nrows
104 +
105 + return {"image": max_image_tokens}
106
107 def get_image_feature_grid_size(
108 self,
@@ -111,32 +128,11 @@ class FuyuProcessingInfo(BaseProcessingInfo):
128 nrows = math.ceil(image_height / patch_height)
129 return ncols, nrows
130
114 - def get_num_image_tokens(
115 - self,
116 - *,
117 - image_width: int,
118 - image_height: int,
119 - ) -> int:
120 - ncols, nrows = self.get_image_feature_grid_size(
121 - image_width=image_width,
122 - image_height=image_height,
123 - )
124 -
125 - return ncols * nrows
126 -
131 def get_image_size_with_most_features(self) -> ImageSize:
132 image_processor = self.get_image_processor()
133 return ImageSize(width=image_processor.size["width"],
134 height=image_processor.size["height"])
135
132 - def get_max_image_tokens(self) -> int:
133 - target_width, target_height = self.get_image_size_with_most_features()
134 -
135 - return self.get_num_image_tokens(
136 - image_width=target_width,
137 - image_height=target_height,
138 - )
139 -
136
137 class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
138
@@ -196,6 +192,19 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
192
193 processed_outputs["image_patches"] = image_patches[0]
194
195 + # get patch grid size for each image
196 + embed_is_patch = []
197 + for image in images:
198 + ncols, nrows = self.info.get_image_feature_grid_size(
199 + image_width=image.width,
200 + image_height=image.height,
201 + )
202 +
203 + mask = torch.tensor(([True] * ncols + [False]) * nrows)
204 + embed_is_patch.append(mask)
205 +
206 + processed_outputs["embed_is_patch"] = embed_is_patch
207 +
208 return processed_outputs
209
210 def _apply_hf_processor_tokens_only(
@@ -215,7 +224,8 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
224 hf_inputs: BatchFeature,
225 hf_processor_mm_kwargs: Mapping[str, object],
226 ) -> Mapping[str, MultiModalFieldConfig]:
218 - return dict(image_patches=MultiModalFieldConfig.batched("image"))
227 + return dict(image_patches=MultiModalFieldConfig.batched("image"),
228 + embed_is_patch=MultiModalFieldConfig.batched("image"))
229
230 def _get_prompt_updates(
231 self,
@@ -242,9 +252,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
252 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
253 [_NEWLINE_TOKEN_ID]) * nrows
254
245 - return PromptUpdateDetails.select_token_id(
246 - image_tokens + [bos_token_id],
247 - embed_token_id=_IMAGE_TOKEN_ID,
255 + return PromptUpdateDetails(
256 + full=image_tokens + [bos_token_id],
257 + features=image_tokens,
258 )
259
260 return [
@@ -319,13 +329,20 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
329 raise ValueError("Incorrect type of image patches. "
330 f"Got type: {type(image_patches)}")
331
332 + embed_is_patch = kwargs.pop("embed_is_patch")
333 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
334 + raise ValueError("Incorrect type of embed_is_patch. "
335 + f"Got type: {type(embed_is_patch)}")
336 +
337 image_patches_flat = flatten_bn(image_patches)
338 + embed_is_patch = flatten_bn(embed_is_patch)
339
340 return FuyuImagePatchInputs(
341 type="image_patches",
342 flat_data=self._validate_pixel_values(
343 flatten_bn(image_patches_flat, concat=True)),
344 patches_per_image=[x.size(0) for x in image_patches_flat],
345 + embed_is_patch=embed_is_patch,
346 )
347
348 return None
@@ -347,7 +364,12 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
364 if image_input is None:
365 return None
366
350 - return self._process_image_input(image_input)
367 + image_features = self._process_image_input(image_input)
368 +
369 + return scatter_patch_features(
370 + image_features,
371 + image_input["embed_is_patch"],
372 + )
373
374 def get_input_embeddings(
375 self,
@@ -357,11 +379,8 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
379 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
380 if multimodal_embeddings is not None:
381 inputs_embeds = merge_multimodal_embeddings(
360 - input_ids,
361 - inputs_embeds,
362 - multimodal_embeddings,
363 - _IMAGE_TOKEN_ID,
364 - )
382 + input_ids, inputs_embeds,
383 + select_patch_features(multimodal_embeddings), _IMAGE_TOKEN_ID)
384 return inputs_embeds
385
386 def forward(
vllm/model_executor/models/gemma3_mm.py
+63 -20
@@ -25,7 +25,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
25 PlaceholderFeaturesInfo,
26 PromptReplacement, PromptTargetMatch,
27 PromptUpdate, PromptUpdateDetails,
28 - find_mm_placeholders,
28 + encode_tokens, find_mm_placeholders,
29 replace_token_matches)
30 # yapf: enable
31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
@@ -36,6 +36,7 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
36 from .siglip import SiglipVisionModel
37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
38 maybe_prefix, merge_multimodal_embeddings)
39 +from .vision import scatter_patch_features, select_patch_features
40
41 logger = init_logger(__name__)
42
@@ -53,6 +54,14 @@ class Gemma3ImagePixelInputs(TypedDict):
54 num_patches: torch.Tensor
55 """Shape: `(batch_size * num_images)`"""
56
57 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
58 + """
59 + A boolean mask indicating which image embeddings correspond
60 + to patch tokens.
61 +
62 + Shape: `(batch_size * num_images, num_embeds)`
63 + """
64 +
65
66 Gemma3ImageInputs = Gemma3ImagePixelInputs
67
@@ -174,7 +183,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
183 if processor is None:
184 processor = self.get_hf_processor()
185
177 - boi_token = processor.boi_token
186 + image_token = processor.boi_token
187
188 num_crops = self.get_num_crops(
189 image_width=image_width,
@@ -183,21 +192,19 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
192 )
193
194 if num_crops == 0:
186 - image_text = boi_token
195 + image_text = image_token
196 else:
188 - crops_image_tokens = " ".join(boi_token for _ in range(num_crops))
197 + crops_image_tokens = " ".join(image_token
198 + for _ in range(num_crops))
199 image_text = (
190 - f"Here is the original image {boi_token} and here are some "
200 + f"Here is the original image {image_token} and here are some "
201 f"crops to help you see better {crops_image_tokens}")
202
193 - repl_full = image_text.replace(boi_token,
203 + repl_full = image_text.replace(image_token,
204 processor.full_image_sequence)
205 + repl_features = repl_full.strip("\n")
206
196 - tokenizer = processor.tokenizer
197 - vocab = tokenizer.get_vocab()
198 - image_token_id = vocab[tokenizer.image_token]
199 -
200 - return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
207 + return PromptUpdateDetails(full=repl_full, features=repl_features)
208
209 def get_num_image_tokens(
210 self,
@@ -206,17 +213,19 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
213 image_height: int,
214 processor: Optional[Gemma3Processor],
215 ) -> int:
209 - if processor is None:
210 - processor = self.get_hf_processor()
211 -
212 - num_crops = self.get_num_crops(
216 + tokenizer = self.get_tokenizer()
217 + image_repl = self.get_image_repl(
218 image_width=image_width,
219 image_height=image_height,
220 processor=processor,
221 )
217 - image_seq_len = processor.image_seq_length
222
219 - return (num_crops + 1) * image_seq_len
223 + image_repl_tokens = encode_tokens(
224 + tokenizer,
225 + image_repl.features,
226 + add_special_tokens=False,
227 + )
228 + return len(image_repl_tokens)
229
230 def get_image_size_with_most_features(self) -> ImageSize:
231 processor = self.get_hf_processor()
@@ -292,6 +301,28 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
301 ]
302 hf_processor = self.info.get_hf_processor(**mm_kwargs)
303
304 + image_repl_features = [
305 + self.info.get_image_repl(image_width=size.width,
306 + image_height=size.height,
307 + processor=hf_processor).features
308 + for size in image_sizes
309 + ]
310 +
311 + tokenizer = self.info.get_tokenizer()
312 + image_repls_feature_tokens = [
313 + tokenizer.encode(image_repl, add_special_tokens=False)
314 + for image_repl in image_repl_features
315 + ]
316 +
317 + vocab = tokenizer.get_vocab()
318 + image_token_id = vocab[tokenizer.image_token]
319 +
320 + embed_is_patch = [
321 + torch.tensor(image_repl_tokens) == image_token_id
322 + for image_repl_tokens in image_repls_feature_tokens
323 + ]
324 + processed_outputs["embed_is_patch"] = embed_is_patch
325 +
326 num_crops = [
327 self.info.get_num_crops(image_width=size.width,
328 image_height=size.height,
@@ -313,6 +344,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
344 pixel_values=MultiModalFieldConfig.flat_from_sizes(
345 "image", num_crops + 1),
346 num_crops=MultiModalFieldConfig.batched("image"),
347 + embed_is_patch=MultiModalFieldConfig.batched("image"),
348 )
349
350 def _get_prompt_updates(
@@ -422,7 +454,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
454 item_idx=p.item_idx,
455 start_idx=repl_orig_idxs[p.start_idx],
456 tokens=p.tokens,
425 - is_embed=p.is_embed,
457 ) for p in placeholders
458 ]
459 for modality, placeholders in repls.items()
@@ -541,6 +572,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
572 self, **kwargs: object) -> Optional[Gemma3ImageInputs]:
573 pixel_values = kwargs.pop("pixel_values", None)
574 num_crops = kwargs.pop("num_crops", None)
575 + embed_is_patch = kwargs.pop("embed_is_patch", None)
576 image_embeds = kwargs.pop("image_embeds", None)
577 assert image_embeds is None, "Gemma3 does not support image_embeds."
578 if pixel_values is None:
@@ -554,13 +586,19 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
586 raise ValueError("Incorrect type of num_crops. "
587 f"Got type: {type(num_crops)}")
588
589 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
590 + raise ValueError("Incorrect type of embed_is_patch. "
591 + f"Got type: {type(embed_is_patch)}")
592 +
593 pixel_values = flatten_bn(pixel_values, concat=True)
594 num_crops = flatten_bn(num_crops, concat=True)
595 + embed_is_patch = flatten_bn(embed_is_patch)
596
597 return Gemma3ImagePixelInputs(
598 type="pixel_values",
599 pixel_values=self._validate_pixel_values(pixel_values),
600 num_patches=num_crops + 1,
601 + embed_is_patch=embed_is_patch,
602 )
603
604 def _image_pixels_to_features(
@@ -597,7 +635,12 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
635 if image_input is None:
636 return None
637
600 - return self._process_image_input(image_input)
638 + image_features = self._process_image_input(image_input)
639 +
640 + return scatter_patch_features(
641 + image_features,
642 + image_input["embed_is_patch"],
643 + )
644
645 def get_input_embeddings(
646 self,
@@ -609,7 +652,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
652 inputs_embeds = merge_multimodal_embeddings(
653 input_ids,
654 inputs_embeds,
612 - multimodal_embeddings,
655 + select_patch_features(multimodal_embeddings),
656 self.config.image_token_index,
657 )
658 return inputs_embeds
vllm/model_executor/models/h2ovl.py
+1 -1
@@ -257,7 +257,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
257 repl_features = IMG_CONTEXT * feature_size
258 repl_full = IMG_START + repl_features + IMG_END
259
260 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
260 + return PromptUpdateDetails(full=repl_full, features=repl_features)
261
262 def resolve_min_max_num(
263 self,
vllm/model_executor/models/idefics3.py
+67 -15
@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 MultiModalDataItems,
42 MultiModalFieldConfig,
43 PromptReplacement, PromptUpdate,
44 - PromptUpdateDetails)
44 + encode_tokens)
45 # yapf: enable
46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 from vllm.sequence import IntermediateTensors
@@ -54,6 +54,7 @@ from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
54 from .llama import LlamaModel
55 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
56 merge_multimodal_embeddings)
57 +from .vision import scatter_patch_features, select_patch_features
58
59
60 class Idefics3ImagePixelInputs(TypedDict):
@@ -68,6 +69,14 @@ class Idefics3ImagePixelInputs(TypedDict):
69 num_patches: torch.Tensor
70 """Shape: `(batch_size * num_images)`"""
71
72 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
73 + """
74 + A boolean mask indicating which image embeddings correspond
75 + to patch tokens.
76 +
77 + Shape: `(batch_size * num_images, num_embeds)`
78 + """
79 +
80
81 class Idefics3ImageEmbeddingInputs(TypedDict):
82 type: Literal["image_embeds"]
@@ -77,6 +86,14 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
86 `hidden_size` must match the hidden size of language model backbone.
87 """
88
89 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
90 + """
91 + A boolean mask indicating which image embeddings correspond
92 + to patch tokens.
93 +
94 + Shape: `(batch_size * num_images, num_embeds)`
95 + """
96 +
97
98 ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
99
@@ -258,16 +275,19 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
275 image_height: int,
276 processor: Optional[Idefics3Processor],
277 ) -> int:
261 - if processor is None:
262 - processor = self.get_hf_processor()
263 -
264 - num_patches = self.get_num_patches(
278 + tokenizer = self.get_tokenizer()
279 + image_repl = self.get_image_repl(
280 image_width=image_width,
281 image_height=image_height,
282 processor=processor,
283 )
284
270 - return num_patches * processor.image_seq_len
285 + image_repl_tokens = encode_tokens(
286 + tokenizer,
287 + image_repl,
288 + add_special_tokens=False,
289 + )
290 + return len(image_repl_tokens)
291
292 def get_image_size_with_most_features(self) -> ImageSize:
293 processor = self.get_hf_processor()
@@ -344,6 +364,28 @@ class Idefics3MultiModalProcessor(
364 ]
365 hf_processor = self.info.get_hf_processor(**mm_kwargs)
366
367 + image_repl_features = [
368 + self.info.get_image_repl(image_width=size.width,
369 + image_height=size.height,
370 + processor=hf_processor)
371 + for size in image_sizes
372 + ]
373 +
374 + tokenizer = self.info.get_tokenizer()
375 + image_repls_feature_tokens = [
376 + tokenizer.encode(image_repl, add_special_tokens=False)
377 + for image_repl in image_repl_features
378 + ]
379 +
380 + vocab = tokenizer.get_vocab()
381 + image_token_id = vocab[hf_processor.image_token.content]
382 +
383 + embed_is_patch = [
384 + torch.tensor(image_repl_tokens) == image_token_id
385 + for image_repl_tokens in image_repls_feature_tokens
386 + ]
387 + processed_outputs["embed_is_patch"] = embed_is_patch
388 +
389 num_patches = [
390 self.info.get_num_patches(
391 image_width=size.width,
@@ -373,6 +415,7 @@ class Idefics3MultiModalProcessor(
415 "image", num_patches),
416 image_embeds=MultiModalFieldConfig.batched("image"),
417 num_patches=MultiModalFieldConfig.batched("image"),
418 + embed_is_patch=MultiModalFieldConfig.batched("image"),
419 )
420
421 def _get_prompt_updates(
@@ -384,22 +427,17 @@ class Idefics3MultiModalProcessor(
427 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
428 image_token = hf_processor.image_token.content
429
387 - def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
430 + def get_replacement_idefics3(item_idx: int) -> str:
431 images = mm_items.get_items("image", ImageProcessorItems)
432
433 image_size = images.get_image_size(item_idx)
434
392 - image_repl = self.info.get_image_repl(
435 + return self.info.get_image_repl(
436 image_width=image_size.width,
437 image_height=image_size.height,
438 processor=hf_processor,
439 )
440
398 - return PromptUpdateDetails.select_text(
399 - image_repl,
400 - embed_text=image_token,
401 - )
402 -
441 return [
442 PromptReplacement(
443 modality="image",
@@ -637,6 +675,13 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
675 if pixel_values is None and image_embeds is None:
676 return None
677
678 + embed_is_patch = kwargs.pop("embed_is_patch")
679 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
680 + raise ValueError("Incorrect type of embed_is_patch. "
681 + f"Got type: {type(embed_is_patch)}")
682 +
683 + embed_is_patch = flatten_bn(embed_is_patch)
684 +
685 if image_embeds is not None:
686 if not isinstance(image_embeds, (torch.Tensor, list)):
687 raise ValueError("Incorrect type of image embeddings. "
@@ -645,6 +690,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
690 return Idefics3ImageEmbeddingInputs(
691 type="image_embeds",
692 data=flatten_bn(image_embeds, concat=True),
693 + embed_is_patch=embed_is_patch,
694 )
695
696 if pixel_values is not None:
@@ -672,6 +718,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
718 pixel_values=self._validate_pixel_values(pixel_values),
719 pixel_attention_mask=pixel_attention_mask,
720 num_patches=num_patches,
721 + embed_is_patch=embed_is_patch,
722 )
723
724 raise AssertionError("This line should be unreachable.")
@@ -707,7 +754,12 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
754 if image_input is None:
755 return None
756
710 - return self._process_image_input(image_input)
757 + image_features = self._process_image_input(image_input)
758 +
759 + return scatter_patch_features(
760 + image_features,
761 + image_input["embed_is_patch"],
762 + )
763
764 def get_input_embeddings(
765 self,
@@ -719,7 +771,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
771 inputs_embeds = merge_multimodal_embeddings(
772 input_ids,
773 inputs_embeds,
722 - multimodal_embeddings,
774 + select_patch_features(multimodal_embeddings),
775 self.config.image_token_id,
776 )
777 return inputs_embeds
vllm/model_executor/models/internvl.py
+40 -3
@@ -39,6 +39,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
39 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
40 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41 maybe_prefix, merge_multimodal_embeddings)
42 +from .vision import scatter_patch_features, select_patch_features
43
44 IMG_START = '<img>'
45 IMG_END = '</img>'
@@ -59,6 +60,14 @@ class InternVLImagePixelInputs(TypedDict):
60 num_patches: torch.Tensor
61 """Shape: `(batch_size * num_images)`"""
62
63 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
64 + """
65 + A boolean mask indicating which image embeddings correspond
66 + to patch tokens.
67 +
68 + Shape: `(batch_size * num_images, num_embeds)`
69 + """
70 +
71
72 class InternVLImageEmbeddingInputs(TypedDict):
73 type: Literal["image_embeds"]
@@ -410,12 +419,24 @@ class BaseInternVLProcessor(ABC):
419 torch.tensor([len(item) for item in pixel_values_lst]),
420 }
421
422 + tokenizer = self.tokenizer
423 + image_token_id = self.image_token_id
424 +
425 + embed_is_patch = list[torch.Tensor]()
426 +
427 for pixel_values in pixel_values_lst:
428 num_patches = pixel_values.shape[0]
429 feature_size = num_patches * self.num_image_token
430
431 image_repl = self.get_image_repl(feature_size, num_patches)
432 + feature_tokens = tokenizer.encode(image_repl.features,
433 + add_special_tokens=False)
434 +
435 text = [t.replace('<image>', image_repl.full, 1) for t in text]
436 + embed_is_patch.append(
437 + torch.tensor(feature_tokens) == image_token_id)
438 +
439 + image_inputs["embed_is_patch"] = embed_is_patch
440
441 text_inputs = self.tokenizer(text)
442
@@ -439,7 +460,7 @@ class InternVLProcessor(BaseInternVLProcessor):
460 repl_features = IMG_CONTEXT * feature_size
461 repl_full = IMG_START + repl_features + IMG_END
462
442 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
463 + return PromptUpdateDetails(full=repl_full, features=repl_features)
464
465
466 class BaseInternVLProcessingInfo(BaseProcessingInfo):
@@ -578,6 +599,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
599 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
600 "image", image_num_patches),
601 image_num_patches=MultiModalFieldConfig.batched("image"),
602 + embed_is_patch=MultiModalFieldConfig.batched("image"),
603 image_embeds=MultiModalFieldConfig.batched("image"),
604 image_token_id=MultiModalFieldConfig.shared("image", num_images),
605 )
@@ -809,6 +831,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
831 self, **kwargs: object) -> Optional[InternVLImageInputs]:
832 pixel_values_flat = kwargs.pop("pixel_values_flat", None)
833 image_num_patches = kwargs.pop("image_num_patches", None)
834 + embed_is_patch = kwargs.pop("embed_is_patch", None)
835 image_embeds = kwargs.pop("image_embeds", None)
836
837 if pixel_values_flat is None and image_embeds is None:
@@ -837,14 +860,20 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
860 raise ValueError("Incorrect type of image_num_patches. "
861 f"Got type: {type(image_num_patches)}")
862
863 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
864 + raise ValueError("Incorrect type of embed_is_patch. "
865 + f"Got type: {type(embed_is_patch)}")
866 +
867 pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
868 image_num_patches = flatten_bn(image_num_patches, concat=True)
869 + embed_is_patch = flatten_bn(embed_is_patch)
870
871 return InternVLImagePixelInputs(
872 type="pixel_values",
873 pixel_values_flat=self._validate_pixel_values(
874 pixel_values_flat),
875 num_patches=image_num_patches,
876 + embed_is_patch=embed_is_patch,
877 )
878
879 raise AssertionError("This line should be unreachable.")
@@ -890,7 +919,15 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
919 if image_input is None:
920 return None
921
893 - return self._process_image_input(image_input)
922 + image_features = self._process_image_input(image_input)
923 +
924 + if image_input["type"] != "pixel_values":
925 + return image_features
926 +
927 + return scatter_patch_features(
928 + image_features,
929 + image_input["embed_is_patch"],
930 + )
931
932 def get_input_embeddings(
933 self,
@@ -904,7 +941,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
941 inputs_embeds = merge_multimodal_embeddings(
942 input_ids,
943 inputs_embeds,
907 - multimodal_embeddings,
944 + select_patch_features(multimodal_embeddings),
945 self.img_context_token_id,
946 )
947 return inputs_embeds
vllm/model_executor/models/llava.py
+49 -6
@@ -32,8 +32,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 ImageSize, MultiModalDataItems)
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, ProcessingCache,
35 - PromptReplacement, PromptUpdate,
36 - PromptUpdateDetails)
35 + PromptReplacement, PromptUpdate)
36 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
38
@@ -43,7 +42,8 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
42 from .siglip import SiglipVisionModel
43 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
44 maybe_prefix, merge_multimodal_embeddings)
46 -from .vision import get_vision_encoder_info
45 +from .vision import (get_vision_encoder_info, scatter_patch_features,
46 + select_patch_features)
47
48
49 class LlavaImagePixelInputs(TypedDict):
@@ -67,6 +67,14 @@ class PixtralHFImagePixelInputs(TypedDict):
67 in which case the data is passed as a list instead of a batched tensor.
68 """
69
70 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
71 + """
72 + A boolean mask indicating which image embeddings correspond
73 + to patch tokens.
74 +
75 + Shape: `(batch_size * num_images, num_embeds)`
76 + """
77 +
78
79 class LlavaImageEmbeddingInputs(TypedDict):
80 type: Literal["image_embeds"]
@@ -335,6 +343,23 @@ class PixtralHFMultiModalProcessor(
343 for p, (h, w) in zip(pixel_values, image_sizes)
344 ]
345
346 + hf_config = self.info.get_hf_config()
347 + vision_config = hf_config.vision_config
348 + assert isinstance(vision_config, PixtralVisionConfig)
349 + encoder_info = PixtralHFEncoderInfo(vision_config)
350 +
351 + tile_sizes = [
352 + encoder_info.get_patch_grid_size(
353 + image_width=pixel_value.shape[-1],
354 + image_height=pixel_value.shape[-2],
355 + ) for pixel_value in processed_outputs["pixel_values"]
356 + ]
357 + embed_is_patch = [
358 + torch.tensor(([True] * ncols + [False]) * nrows)
359 + for ncols, nrows in tile_sizes
360 + ]
361 + processed_outputs["embed_is_patch"] = embed_is_patch
362 +
363 return processed_outputs
364
365 def _get_mm_fields_config(
@@ -344,6 +369,7 @@ class PixtralHFMultiModalProcessor(
369 ) -> Mapping[str, MultiModalFieldConfig]:
370 return dict(
371 pixel_values=MultiModalFieldConfig.batched("image"),
372 + embed_is_patch=MultiModalFieldConfig.batched("image"),
373 image_embeds=MultiModalFieldConfig.batched("image"),
374 )
375
@@ -378,7 +404,7 @@ class PixtralHFMultiModalProcessor(
404 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
405 tokens[-1] = image_end_id
406
381 - return PromptUpdateDetails.select_token_id(tokens, image_token_id)
407 + return tokens
408
409 return [
410 PromptReplacement(
@@ -586,9 +612,17 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
612 f"Got type: {type(pixel_values)}")
613
614 if self.config.vision_config.model_type == "pixtral":
615 + embed_is_patch = kwargs.pop("embed_is_patch")
616 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
617 + raise ValueError("Incorrect type of embed_is_patch. "
618 + f"Got type: {type(embed_is_patch)}")
619 +
620 + embed_is_patch = flatten_bn(embed_is_patch)
621 +
622 return PixtralHFImagePixelInputs(
623 type="pixel_values_pixtral",
624 pixel_values=flatten_bn(pixel_values),
625 + embed_is_patch=embed_is_patch,
626 )
627
628 return LlavaImagePixelInputs(
@@ -680,7 +714,16 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
714 if image_input is None:
715 return None
716
683 - return self._process_image_input(image_input)
717 + image_features = self._process_image_input(image_input)
718 +
719 + if image_input["type"] != "pixel_values_pixtral":
720 + # The path is used for pixtral (V0 only) and llava (V0/V1)
721 + return image_features
722 +
723 + return scatter_patch_features(
724 + image_features,
725 + image_input["embed_is_patch"],
726 + )
727
728 def get_input_embeddings(
729 self,
@@ -692,7 +735,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
735 inputs_embeds = merge_multimodal_embeddings(
736 input_ids,
737 inputs_embeds,
695 - multimodal_embeddings,
738 + select_patch_features(multimodal_embeddings),
739 self.config.image_token_index,
740 )
741 return inputs_embeds
vllm/model_executor/models/minicpmo.py
+65 -9
@@ -40,8 +40,7 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,
40 DictEmbeddingItems, ModalityData,
41 ModalityDataItems, MultiModalDataItems,
42 MultiModalDataParser)
43 -from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
44 - PromptUpdateDetails)
43 +from vllm.multimodal.processing import PromptReplacement, PromptUpdate
44 from vllm.multimodal.profiling import ProcessorInputs
45
46 from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,
@@ -51,6 +50,7 @@ from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,
50 _minicpmv_field_config)
51 from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn,
52 maybe_prefix)
53 +from .vision import scatter_patch_features
54
55 CPU_DEVICE = torch.device("cpu")
56
@@ -73,6 +73,14 @@ class MiniCPMOAudioFeatureInputs(TypedDict):
73 which equals to `audio_features.shape[-1]`
74 """
75
76 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
77 + """
78 + A boolean mask indicating which audio embeddings correspond
79 + to patch tokens.
80 +
81 + Shape: `(batch_size * num_audios, num_embeds)`
82 + """
83 +
84
85 class MiniCPMOAudioEmbeddingInputs(TypedDict):
86 type: Literal["audio_embeds"]
@@ -85,6 +93,14 @@ class MiniCPMOAudioEmbeddingInputs(TypedDict):
93 Length of each slice may vary, so pass it as a list.
94 """
95
96 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
97 + """
98 + A boolean mask indicating which audio embeddings correspond
99 + to patch tokens.
100 +
101 + Shape: `(batch_size * num_audios, num_embeds)`
102 + """
103 +
104
105 MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,
106 MiniCPMOAudioEmbeddingInputs]
@@ -99,6 +115,7 @@ def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
115 audio_features=MultiModalFieldConfig.batched("audio"),
116 audio_feature_lens=MultiModalFieldConfig.batched("audio"),
117 audio_embeds=MultiModalFieldConfig.batched("audio"),
118 + audio_embed_is_patch=MultiModalFieldConfig.batched("audio"),
119 audio_token_id=MultiModalFieldConfig.shared("audio", num_audios),
120 )
121
@@ -180,7 +197,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
197 pool_step = self.get_default_audio_pool_step()
198 fbank_feat_in_chunk = 100
199 cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
183 - return (cnn_feat_in_chunk - pool_step) // pool_step + 1
200 + num_audio_tokens = (cnn_feat_in_chunk - pool_step) // pool_step + 1
201 + return num_audio_tokens + 2 # <audio>(<unk>*N)</audio>
202
203 def get_max_audio_chunks_with_most_features(self) -> int:
204 return 30
@@ -191,7 +209,8 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
209
210 def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:
211 sampling_rate = self.get_default_audio_sampling_rate()
194 - num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk()
212 + # exclude <audio> </audio>
213 + num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk() - 2
214 return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1
215
216 def get_num_frames_with_most_features(
@@ -276,6 +295,13 @@ class MiniCPMOMultiModalProcessor(
295
296 if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):
297 audio_inputs = {}
298 +
299 + audio_lens = [
300 + self.info.get_audio_len_by_num_chunks(
301 + sum(map(len,
302 + parsed_audios.get(i)["audio_embeds"])))
303 + for i in range(len(parsed_audios))
304 + ]
305 else:
306 audio_inputs = self._base_call_hf_processor(
307 prompts=[self.info.audio_pattern] * len(parsed_audios),
@@ -297,7 +323,27 @@ class MiniCPMOMultiModalProcessor(
323 ]
324 audio_inputs["audio_features"] = unpadded_audio_features
325
326 + audio_lens = [
327 + parsed_audios.get_audio_length(i)
328 + for i in range(len(parsed_audios))
329 + ]
330 +
331 + audio_repl_features = [
332 + self.get_audio_prompt_texts(audio_len) for audio_len in audio_lens
333 + ]
334 +
335 tokenizer = self.info.get_tokenizer()
336 + audio_repls_feature_tokens = [
337 + tokenizer.encode(audio_repl, add_special_tokens=False)
338 + for audio_repl in audio_repl_features
339 + ]
340 +
341 + embed_is_patch = [
342 + self.get_embed_is_patch(audio_repl_tokens)
343 + for audio_repl_tokens in audio_repls_feature_tokens
344 + ]
345 + audio_inputs["audio_embed_is_patch"] = embed_is_patch
346 +
347 unk_token_id = tokenizer.get_vocab()["<unk>"]
348 audio_inputs["audio_token_id"] = torch.tensor(unk_token_id)
349
@@ -338,10 +384,7 @@ class MiniCPMOMultiModalProcessor(
384 else:
385 audio_len = audios.get_audio_length(item_idx)
386
341 - return PromptUpdateDetails.select_text(
342 - self.get_audio_prompt_texts(audio_len),
343 - "<unk>",
344 - )
387 + return self.get_audio_prompt_texts(audio_len)
388
389 return [
390 *base_updates,
@@ -670,6 +713,13 @@ class MiniCPMO(MiniCPMV2_6):
713 assert isinstance(audio_token_id, torch.Tensor)
714 self.mm_token_ids.add(audio_token_id.flatten().unique().item())
715
716 + audio_embed_is_patch = kwargs.pop("audio_embed_is_patch")
717 + if not isinstance(audio_embed_is_patch, (torch.Tensor, list)):
718 + raise ValueError("Incorrect type of audio_embed_is_patch. "
719 + f"Got type: {type(audio_embed_is_patch)}")
720 +
721 + audio_embed_is_patch = flatten_bn(audio_embed_is_patch)
722 +
723 if audio_embeds is not None:
724 if not isinstance(audio_embeds, (torch.Tensor, list)):
725 raise ValueError("Incorrect type of audio_embeds. "
@@ -680,6 +730,7 @@ class MiniCPMO(MiniCPMV2_6):
730 return MiniCPMOAudioEmbeddingInputs(
731 type="audio_embeds",
732 audio_embeds=audio_embeds_flat,
733 + embed_is_patch=audio_embed_is_patch,
734 )
735
736 if not isinstance(audio_features, (torch.Tensor, list)):
@@ -698,6 +749,7 @@ class MiniCPMO(MiniCPMV2_6):
749 type="audio_features",
750 audio_features=audio_features_flat,
751 audio_feature_lens=audio_feature_lens_flat,
752 + embed_is_patch=audio_embed_is_patch,
753 )
754
755 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
@@ -729,6 +781,10 @@ class MiniCPMO(MiniCPMV2_6):
781 if modality == "audios":
782 audio_input = modalities["audios"]
783 audio_features = self._process_audio_input(audio_input)
732 - multimodal_embeddings += tuple(audio_features)
784 + multimodal_embeddings += tuple(
785 + scatter_patch_features(
786 + audio_features,
787 + audio_input["embed_is_patch"],
788 + ))
789
790 return multimodal_embeddings
vllm/model_executor/models/minicpmv.py
+101 -41
@@ -56,7 +56,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,
56 VideoItem, VideoProcessorItems)
57 from vllm.multimodal.processing import (BaseMultiModalProcessor,
58 BaseProcessingInfo, PromptReplacement,
59 - PromptUpdate, PromptUpdateDetails)
59 + PromptUpdate)
60 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
61 from vllm.platforms import current_platform
62 from vllm.sequence import IntermediateTensors
@@ -67,6 +67,7 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
67 SupportsMultiModal, SupportsPP)
68 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
69 merge_multimodal_embeddings)
70 +from .vision import scatter_patch_features, select_patch_features
71
72 # For profile run
73 _MAX_FRAMES_PER_VIDEO = 16
@@ -89,6 +90,14 @@ class MiniCPMVImagePixelInputs(TypedDict):
90 This should be in `(height, width)` format.
91 """
92
93 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
94 + """
95 + A boolean mask indicating which image embeddings correspond
96 + to patch tokens.
97 +
98 + Shape: `(batch_size * num_images, num_embeds)`
99 + """
100 +
101 num_slices: torch.Tensor
102 """Shape: `(batch_size * num_images)`"""
103
@@ -103,6 +112,14 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):
112 instead of a batched tensor.
113 """
114
115 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
116 + """
117 + A boolean mask indicating which image embeddings correspond
118 + to patch tokens.
119 +
120 + Shape: `(batch_size * num_images, num_embeds)`
121 + """
122 +
123
124 MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,
125 MiniCPMVImageEmbeddingInputs]
@@ -228,10 +245,12 @@ def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
245 image_sizes=MultiModalFieldConfig.batched("image"),
246 tgt_sizes=MultiModalFieldConfig.batched("image"),
247 image_embeds=MultiModalFieldConfig.batched("image"),
248 + embed_is_patch=MultiModalFieldConfig.batched("image"),
249 video_pixel_values=MultiModalFieldConfig.batched("video"),
250 video_image_sizes=MultiModalFieldConfig.batched("video"),
251 video_tgt_sizes=MultiModalFieldConfig.batched("video"),
252 video_embeds=MultiModalFieldConfig.batched("video"),
253 + video_embed_is_patch=MultiModalFieldConfig.batched("video"),
254 image_token_id=MultiModalFieldConfig.shared("image", num_images),
255 video_token_id=MultiModalFieldConfig.shared("video", num_videos),
256 )
@@ -379,43 +398,22 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
398 use_image_id=use_image_id,
399 )
400
382 - def get_sliced_grid(
383 - self,
384 - image_size: ImageSize,
385 - # For MiniCPM V/O 2.6
386 - max_slice_nums: Optional[int] = None,
387 - ) -> Optional[tuple[int, int]]:
388 - image_processor = self.get_image_processor()
389 - version = self.get_model_version()
390 -
391 - if version == (2, 0) or version == (2, 5):
392 - return image_processor.get_sliced_grid(image_size)
393 -
394 - if max_slice_nums is None:
395 - max_slice_nums = image_processor.max_slice_nums
396 -
397 - return image_processor.get_sliced_grid(
398 - image_size,
399 - max_slice_nums=max_slice_nums,
400 - )
401 -
401 def get_num_image_tokens(
402 self,
403 image_size: ImageSize,
404 max_slice_nums: Optional[int] = None,
405 + use_image_id: bool = True,
406 ) -> int:
407 - image_processor = self.get_image_processor()
408 -
409 - grid = self.get_sliced_grid(
407 + tokenizer = self.get_tokenizer()
408 + image_placeholders = self.get_slice_image_placeholder(
409 image_size,
410 max_slice_nums=max_slice_nums,
411 + use_image_id=use_image_id,
412 )
413 - if grid is None:
414 - ncols = nrows = 0
415 - else:
416 - ncols, nrows = grid
413 + image_token_ids = tokenizer.encode(image_placeholders,
414 + add_special_tokens=False)
415
418 - return (ncols * nrows + 1) * image_processor.image_feature_size
416 + return len(image_token_ids)
417
418 def get_max_image_tokens(self) -> int:
419 image_size = self.get_image_size_with_most_features()
@@ -435,6 +433,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
433 return self.get_num_image_tokens(
434 frame_size,
435 max_slice_nums=self.get_video_max_slice_num(),
436 + use_image_id=False,
437 )
438
439 def get_max_video_tokens(
@@ -540,6 +539,14 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
539 use_image_id=False,
540 ) * num_frames
541
542 + def get_embed_is_patch(
543 + self,
544 + input_ids: list[int],
545 + ) -> torch.Tensor:
546 + tokenizer = self.info.get_tokenizer()
547 + unk_token_id = tokenizer.get_vocab()["<unk>"]
548 + return torch.tensor(input_ids) == unk_token_id
549 +
550 def process_images(
551 self,
552 mm_data: Mapping[str, object],
@@ -563,7 +570,26 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
570 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
571 )
572
573 + image_sizes = [
574 + parsed_images.get_image_size(i) for i in range(len(parsed_images))
575 + ]
576 + image_repl_features = [
577 + self.get_image_prompt_texts(size, idx)
578 + for idx, size in enumerate(image_sizes)
579 + ]
580 +
581 tokenizer = self.info.get_tokenizer()
582 + image_repls_feature_tokens = [
583 + tokenizer.encode(image_repl, add_special_tokens=False)
584 + for image_repl in image_repl_features
585 + ]
586 +
587 + embed_is_patch = [
588 + self.get_embed_is_patch(image_repl_tokens)
589 + for image_repl_tokens in image_repls_feature_tokens
590 + ]
591 + image_inputs["embed_is_patch"] = embed_is_patch
592 +
593 unk_token_id = tokenizer.get_vocab()["<unk>"]
594 image_inputs["image_token_id"] = torch.tensor(unk_token_id)
595
@@ -599,9 +625,31 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
625 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
626 )
627
602 - video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
628 + frame_sizes = [
629 + parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))
630 + ]
631 + num_frames = [
632 + parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))
633 + ]
634 + video_repl_features = [
635 + self.get_video_prompt_texts(size, nframes)
636 + for size, nframes in zip(frame_sizes, num_frames)
637 + ]
638
639 tokenizer = self.info.get_tokenizer()
640 + video_repls_feature_tokens = [
641 + tokenizer.encode(video_repl, add_special_tokens=False)
642 + for video_repl in video_repl_features
643 + ]
644 +
645 + embed_is_patch = [
646 + self.get_embed_is_patch(video_repl_tokens)
647 + for video_repl_tokens in video_repls_feature_tokens
648 + ]
649 + video_inputs["embed_is_patch"] = embed_is_patch
650 +
651 + video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
652 +
653 unk_token_id = tokenizer.get_vocab()["<unk>"]
654 video_inputs["video_token_id"] = torch.tensor(unk_token_id)
655
@@ -692,10 +740,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
740
741 image_size = images.get_image_size(item_idx)
742
695 - return PromptUpdateDetails.select_text(
696 - self.get_image_prompt_texts(image_size, item_idx),
697 - "<unk>",
698 - )
743 + return self.get_image_prompt_texts(image_size, item_idx)
744
745 def get_video_replacement(item_idx: int):
746 videos = mm_items.get_items(
@@ -704,10 +749,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
749 frame_size = videos.get_frame_size(item_idx)
750 num_frames = videos.get_num_frames(item_idx)
751
707 - return PromptUpdateDetails.select_text(
708 - self.get_video_prompt_texts(frame_size, num_frames),
709 - "<unk>",
710 - )
752 + return self.get_video_prompt_texts(frame_size, num_frames)
753
754 get_replacement = {
755 "image": get_image_replacement,
@@ -790,6 +832,14 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
832 assert isinstance(image_token_id, torch.Tensor)
833 self.mm_token_ids.add(image_token_id.flatten().unique().item())
834
835 + embed_is_patch = kwargs.pop("embed_is_patch")
836 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
837 + raise ValueError(
838 + f"Incorrect type of embed_is_patch for {modality=}. "
839 + f"Got type: {type(embed_is_patch)}")
840 +
841 + embed_is_patch = flatten_bn(embed_is_patch)
842 +
843 if image_embeds is not None:
844 if not isinstance(image_embeds, (torch.Tensor, list)):
845 raise ValueError(
@@ -801,6 +851,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
851 return MiniCPMVImageEmbeddingInputs(
852 type="image_embeds",
853 image_embeds=image_embeds_flat,
854 + embed_is_patch=embed_is_patch,
855 )
856
857 if not isinstance(pixel_values, (torch.Tensor, list)):
@@ -828,6 +879,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
879 type="pixel_values",
880 pixel_values=pixel_values_flat,
881 tgt_sizes=tgt_sizes_flat,
882 + embed_is_patch=embed_is_patch,
883 num_slices=num_slices_flat,
884 )
885
@@ -884,11 +936,19 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
936 if modality == "images":
937 image_input = modalities["images"]
938 image_features = self._process_vision_input(image_input)
887 - multimodal_embeddings += tuple(image_features)
939 + multimodal_embeddings += tuple(
940 + scatter_patch_features(
941 + image_features,
942 + image_input["embed_is_patch"],
943 + ))
944 if modality == "videos":
945 video_input = modalities["videos"]
946 video_features = self._process_vision_input(video_input)
891 - multimodal_embeddings += tuple(video_features)
947 + multimodal_embeddings += tuple(
948 + scatter_patch_features(
949 + video_features,
950 + video_input["embed_is_patch"],
951 + ))
952
953 return multimodal_embeddings
954
@@ -911,7 +971,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
971 inputs_embeds = merge_multimodal_embeddings(
972 input_ids,
973 inputs_embeds,
914 - multimodal_embeddings,
974 + select_patch_features(multimodal_embeddings),
975 list(self.mm_token_ids),
976 )
977 return inputs_embeds
vllm/model_executor/models/mistral3.py
+44 -6
@@ -27,8 +27,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
27 MultiModalDataItems)
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 BaseProcessingInfo, ProcessingCache,
30 - PromptReplacement, PromptUpdate,
31 - PromptUpdateDetails)
30 + PromptReplacement, PromptUpdate)
31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
32 from vllm.sequence import IntermediateTensors
33
@@ -36,7 +35,8 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
35 from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
36 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
37 maybe_prefix, merge_multimodal_embeddings)
39 -from .vision import get_vision_encoder_info
38 +from .vision import (get_vision_encoder_info, scatter_patch_features,
39 + select_patch_features)
40
41
42 class Mistral3ImagePixelInputs(TypedDict):
@@ -49,6 +49,14 @@ class Mistral3ImagePixelInputs(TypedDict):
49 in which case the data is passed as a list instead of a batched tensor.
50 """
51
52 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
53 + """
54 + A boolean mask indicating which image embeddings correspond
55 + to patch tokens.
56 +
57 + Shape: `(batch_size, num_images, num_embeds)`
58 + """
59 +
60
61 class Mistral3PatchMerger(nn.Module):
62 """
@@ -258,6 +266,23 @@ class Mistral3MultiModalProcessor(
266 p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)
267 ]
268
269 + hf_config = self.info.get_hf_config()
270 + vision_config = hf_config.vision_config
271 + assert isinstance(vision_config, PixtralVisionConfig)
272 + encoder_info = PixtralHFEncoderInfo(vision_config)
273 +
274 + tile_sizes = [
275 + encoder_info.get_patch_grid_size(
276 + image_width=pixel_value.shape[-1],
277 + image_height=pixel_value.shape[-2],
278 + ) for pixel_value in processed_outputs["pixel_values"]
279 + ]
280 + embed_is_patch = [
281 + torch.tensor(([True] * ncols + [False]) * nrows)
282 + for ncols, nrows in tile_sizes
283 + ]
284 + processed_outputs["embed_is_patch"] = embed_is_patch
285 +
286 return processed_outputs
287
288 def _get_mm_fields_config(
@@ -267,6 +292,7 @@ class Mistral3MultiModalProcessor(
292 ) -> Mapping[str, MultiModalFieldConfig]:
293 return dict(
294 pixel_values=MultiModalFieldConfig.batched("image"),
295 + embed_is_patch=MultiModalFieldConfig.batched("image"),
296 image_embeds=MultiModalFieldConfig.batched("image"),
297 )
298
@@ -301,7 +327,7 @@ class Mistral3MultiModalProcessor(
327 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
328 tokens[-1] = image_end_id
329
304 - return PromptUpdateDetails.select_token_id(tokens, image_token_id)
330 + return tokens
331
332 return [
333 PromptReplacement(
@@ -392,6 +418,8 @@ def init_vision_tower_for_llava(
418 )
419
420
421 +# TODO(mgoin): Support V1, there are issues with image batching/chunking
422 +# that need to be resolved first.
423 @MULTIMODAL_REGISTRY.register_processor(
424 _build_mistral3_processor,
425 info=_build_mistral3_info,
@@ -481,9 +509,16 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
509 raise ValueError("Incorrect type of pixel values. "
510 f"Got type: {type(pixel_values)}")
511
512 + assert self.config.vision_config.model_type == "pixtral"
513 + embed_is_patch = kwargs.pop("embed_is_patch")
514 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
515 + raise ValueError("Incorrect type of embed_is_patch. "
516 + f"Got type: {type(embed_is_patch)}")
517 +
518 return Mistral3ImagePixelInputs(
519 type="pixel_values_pixtral",
520 pixel_values=flatten_bn(pixel_values),
521 + embed_is_patch=flatten_bn(embed_is_patch),
522 )
523
524 def _process_image_input(
@@ -522,7 +557,10 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
557
558 vision_embeddings = self._process_image_input(image_input)
559
525 - return vision_embeddings
560 + return scatter_patch_features(
561 + vision_embeddings,
562 + image_input["embed_is_patch"],
563 + )
564
565 def get_input_embeddings(
566 self,
@@ -534,7 +572,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,
572 inputs_embeds = merge_multimodal_embeddings(
573 input_ids,
574 inputs_embeds,
537 - multimodal_embeddings,
575 + select_patch_features(multimodal_embeddings),
576 self.config.image_token_index,
577 )
578 return inputs_embeds
vllm/model_executor/models/molmo.py
+61 -13
@@ -46,8 +46,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
46 MultiModalDataItems)
47 from vllm.multimodal.processing import (BaseMultiModalProcessor,
48 BaseProcessingInfo, PromptIndexTargets,
49 - PromptInsertion, PromptUpdate,
50 - PromptUpdateDetails)
49 + PromptInsertion, PromptUpdate)
50 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
51 from vllm.sequence import IntermediateTensors
52
@@ -57,6 +56,7 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
56 is_pp_missing_parameter,
57 make_empty_intermediate_tensors_factory, make_layers,
58 maybe_prefix, merge_multimodal_embeddings)
59 +from .vision import scatter_patch_features, select_patch_features
60
61 # TODO: hard-coded for now. Consider making it configurable.
62 VIT_LAYERS = [-2, -9]
@@ -84,6 +84,14 @@ class MolmoImageInputs(TypedDict):
84 Shape: `(batch_size * num_images, num_crops, num_patch)`
85 """
86
87 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
88 + """
89 + A boolean mask indicating which image embeddings correspond
90 + to patch tokens.
91 +
92 + Shape: `(batch_size * num_images, num_embeds)`
93 + """
94 +
95 num_crops: torch.Tensor
96 """Shape: `(batch_size * num_images)`"""
97
@@ -1138,6 +1146,30 @@ class MolmoProcessorWrapper:
1146 if image_input_idx is not None:
1147 feat_is_patch = image_input_idx >= 0
1148
1149 + input_is_embed = torch.isin(
1150 + input_ids,
1151 + torch.tensor([
1152 + self.image_patch_id,
1153 + self.im_col_id,
1154 + self.im_start_id,
1155 + self.im_end_id,
1156 + ]),
1157 + )
1158 + embed_ids = input_ids[input_is_embed]
1159 + embed_is_patch = embed_ids == self.image_patch_id
1160 + assert embed_is_patch.sum() == feat_is_patch.sum()
1161 +
1162 + # image_tokens = extra_joint + joint
1163 + # Both `extra_joint` and `joint` have `im_start_id` and `im_end_id`
1164 + embed_start = torch.nonzero(embed_ids == self.im_start_id)[::2, 0]
1165 + embed_end = torch.nonzero(embed_ids == self.im_end_id)[1::2, 0]
1166 + assert len(embed_start) == len(embed_end) == len(images)
1167 +
1168 + embed_is_patch = [
1169 + embed_is_patch[start:end + 1]
1170 + for start, end in zip(embed_start, embed_end)
1171 + ]
1172 +
1173 tilings = [
1174 self.select_tiling(
1175 image_width=image.size[0],
@@ -1149,6 +1181,7 @@ class MolmoProcessorWrapper:
1181 assert num_crops.sum() == len(feat_is_patch)
1182
1183 outputs["feat_is_patch"] = feat_is_patch
1184 + outputs["embed_is_patch"] = embed_is_patch
1185 outputs["num_crops"] = num_crops
1186 outputs["img_patch_id"] = self.image_patch_id
1187
@@ -1187,13 +1220,17 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1220 )
1221 pooling_size = processor.pooling_size
1222
1190 - image_token_length_w = processor.image_token_length_w
1191 - image_token_length_h = processor.image_token_length_h
1223 + base_image_input_size = processor.base_image_input_size
1224 + base_image_input_d = processor.image_patch_size
1225 +
1226 + crop_patches = base_image_input_size[0] // base_image_input_d
1227
1193 - extra = image_token_length_w * image_token_length_h
1194 - joint = ((ncols + 1) // pooling_size) * ((nrows + 1) // pooling_size)
1228 + per_row = ncols // pooling_size + 1
1229 + joint = per_row * (nrows // pooling_size) + 2
1230 + image_token_length = (crop_patches + pooling_size - 1) // pooling_size
1231 + resize = (image_token_length + 1) * image_token_length + 2
1232
1196 - return extra + joint
1233 + return resize + joint
1234
1235 def get_max_image_tokens(self) -> int:
1236 target_width, target_height = self.get_image_size_with_most_features()
@@ -1291,6 +1328,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1328 "image", num_crops),
1329 feat_is_patch=MultiModalFieldConfig.flat_from_sizes(
1330 "image", num_crops),
1331 + embed_is_patch=MultiModalFieldConfig.batched("image"),
1332 num_crops=MultiModalFieldConfig.batched("image"),
1333 img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1334 )
@@ -1330,10 +1368,8 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1368 joint = ([img_start_id] + joint_row *
1369 ((nrows + 1) // pooling_size) + [img_end_id])
1370
1333 - return PromptUpdateDetails.select_token_id(
1334 - extra_joint + joint,
1335 - embed_token_id=img_patch_id,
1336 - )
1371 + image_tokens = extra_joint + joint
1372 + return image_tokens
1373
1374 return [
1375 PromptInsertion(
@@ -1439,6 +1475,11 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1475 raise ValueError("Incorrect type of feat_is_patch. "
1476 f"Got type: {type(feat_is_patch)}")
1477
1478 + embed_is_patch = kwargs.pop("embed_is_patch", None)
1479 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
1480 + raise ValueError("Incorrect type of embed_is_patch. "
1481 + f"Got type: {type(embed_is_patch)}")
1482 +
1483 num_crops = kwargs.pop("num_crops", None)
1484 if not isinstance(num_crops, (torch.Tensor, list)):
1485 raise ValueError("Incorrect type of num_crops. "
@@ -1450,12 +1491,14 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1491 f"Got type: {type(img_patch_id)}")
1492 self.img_patch_id = img_patch_id.flatten().unique().item()
1493
1494 + embed_is_patch = flatten_bn(embed_is_patch)
1495 num_crops = flatten_bn(num_crops, concat=True)
1496
1497 return MolmoImageInputs(
1498 images=images,
1499 image_masks=image_masks,
1500 feat_is_patch=feat_is_patch,
1501 + embed_is_patch=embed_is_patch,
1502 num_crops=num_crops,
1503 )
1504
@@ -1494,7 +1537,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1537 if image_input is None:
1538 return None
1539
1497 - return self._process_image_input(image_input)
1540 + image_features = self._process_image_input(image_input)
1541 +
1542 + return scatter_patch_features(
1543 + image_features,
1544 + image_input["embed_is_patch"],
1545 + )
1546
1547 def get_input_embeddings(
1548 self,
@@ -1508,7 +1556,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1556 inputs_embeds = merge_multimodal_embeddings(
1557 input_ids,
1558 inputs_embeds,
1511 - multimodal_embeddings,
1559 + select_patch_features(multimodal_embeddings),
1560 self.img_patch_id,
1561 )
1562 return inputs_embeds
vllm/model_executor/models/nvlm_d.py
+30 -2
@@ -57,7 +57,7 @@ class NVLMProcessor(BaseInternVLProcessor):
57 # when trying to find "<tile" as a subsequence of "<Image><tile"
58 repl = "<Image>" + features + "</Image>"
59
60 - return PromptUpdateDetails.select_text(repl, IMG_PAD)
60 + return PromptUpdateDetails(full=repl, features=repl)
61
62
63 class NVLMProcessingInfo(BaseInternVLProcessingInfo):
@@ -84,6 +84,31 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):
84 **kwargs,
85 )
86
87 + def get_max_image_tokens(self) -> int:
88 + hf_processor = self.get_hf_processor()
89 + tokenizer = hf_processor.tokenizer
90 +
91 + max_num_patches = hf_processor.max_dynamic_patch
92 + # we need +1 here because max_dynamic_patch in config doesn't
93 + # include the thumbnail patch
94 + tile_pos_identifiers = [
95 + f"<tile_{i+1}>" for i in range(max_num_patches)
96 + ]
97 + if hf_processor.use_thumbnail and max_num_patches != 1:
98 + tile_pos_identifiers += ["<tile_global_thumbnail>"]
99 +
100 + # "<Image><tile" is tokenized as ["<Image", "><", "tile"]
101 + # so we include <tile_1> in the start_str
102 + start_str = "<Image>" + tile_pos_identifiers.pop(0)
103 + end_str = "</Image>"
104 + start_token_len = len(tokenizer.encode(start_str))
105 + end_token_len = len(tokenizer.encode(end_str))
106 + tile_token_len = sum(
107 + len(tokenizer.encode(identifier))
108 + for identifier in tile_pos_identifiers)
109 + non_image_tokens_num = start_token_len + end_token_len + tile_token_len
110 + return super().get_max_image_tokens() + non_image_tokens_num
111 +
112
113 class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
114
@@ -152,7 +177,10 @@ class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
177
178 repl = hf_processor.get_image_repl(feature_size, num_patches)
179
155 - return PromptUpdateDetails.select_text(repl.full + "\n", IMG_PAD)
180 + return PromptUpdateDetails(
181 + full=repl.full + "\n",
182 + features=repl.features + "\n",
183 + )
184
185 # See note in dummy data regarding why we have the extra newline
186 return [
vllm/model_executor/models/paligemma.py
+3 -3
@@ -162,9 +162,9 @@ class PaliGemmaMultiModalProcessor(
162 modality="image",
163 target=PromptIndexTargets.prefix(
164 [bos_token_id] if tokenizer.add_bos_token else []),
165 - insertion=PromptUpdateDetails.select_token_id(
166 - image_tokens + [bos_token_id],
167 - embed_token_id=image_token_id,
165 + insertion=PromptUpdateDetails(
166 + full=image_tokens + [bos_token_id],
167 + features=image_tokens,
168 ),
169 )
170 ]
vllm/model_executor/models/phi3v.py
+8 -3
@@ -40,7 +40,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
40 from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 BaseProcessingInfo, BoundPromptUpdate,
42 PlaceholderFeaturesInfo,
43 - PromptReplacement, PromptUpdate)
43 + PromptReplacement, PromptUpdate,
44 + PromptUpdateDetails)
45 # yapf: enable
46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 from vllm.sequence import IntermediateTensors
@@ -442,7 +443,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
443 processor=hf_processor,
444 )
445
445 - return [_IMAGE_TOKEN_ID] * num_image_tokens
446 + image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
447 +
448 + return PromptUpdateDetails(
449 + full=image_tokens,
450 + features=image_tokens,
451 + )
452
453 num_images = mm_items.get_count("image", strict=False)
454
@@ -511,7 +517,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
517 item_idx=p.item_idx,
518 start_idx=p.start_idx - 1,
519 tokens=p.tokens,
514 - is_embed=p.is_embed,
520 ) for p in ps
521 ]
522 for modality, ps in placeholders.items()
vllm/model_executor/models/pixtral.py
+40 -8
@@ -37,7 +37,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37 MultiModalDataItems)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 BaseProcessingInfo, PromptReplacement,
40 - PromptUpdate, PromptUpdateDetails)
40 + PromptUpdate)
41 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42 from vllm.sequence import IntermediateTensors
43 from vllm.transformers_utils.tokenizer import (MistralTokenizer,
@@ -46,7 +46,8 @@ from vllm.transformers_utils.tokenizer import (MistralTokenizer,
46 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
47 from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,
48 merge_multimodal_embeddings)
49 -from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs
49 +from .vision import (VisionEncoderInfo, resolve_visual_encoder_outputs,
50 + scatter_patch_features, select_patch_features)
51
52 try:
53 from xformers import ops as xops
@@ -67,6 +68,14 @@ class PixtralImagePixelInputs(TypedDict):
68 The result of stacking :attr:`ImageEncoding.tokens` from each prompt.
69 """
70
71 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
72 + """
73 + A boolean mask indicating which image embeddings correspond
74 + to patch tokens.
75 +
76 + Shape: `(batch_size * num_images, num_embeds)`
77 + """
78 +
79
80 class PixtralProcessorAdapter:
81 """
@@ -135,8 +144,11 @@ class PixtralProcessorAdapter:
144 "For more info, see: "
145 "https://github.com/vllm-project/vllm/issues/8411.")
146
147 + image_token_id = self.image_token_id
148 +
149 images_processed = list[torch.Tensor]()
150 images_tokens = list[torch.Tensor]()
151 + images_embed_is_patch = list[torch.Tensor]()
152
153 for image in images:
154 image_inputs = self.image_processor(ImageChunk(image=image))
@@ -145,10 +157,12 @@ class PixtralProcessorAdapter:
157
158 images_processed.append(image_processed)
159 images_tokens.append(image_tokens)
160 + images_embed_is_patch.append(image_tokens == image_token_id)
161
162 return {
163 "input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),
164 "images": images_processed,
165 + "embed_is_patch": images_embed_is_patch,
166 }
167
168
@@ -199,7 +213,7 @@ class PixtralProcessingInfo(BaseProcessingInfo):
213 ncols, nrows = processor.image_processor._image_to_num_tokens(
214 Image.new("RGB", (image_width, image_height)))
215
202 - return ncols * nrows
216 + return (ncols + 1) * nrows
217
218 def get_image_size_with_most_features(self) -> ImageSize:
219 image_processor = self.get_hf_processor().image_processor
@@ -249,7 +263,10 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
263 hf_inputs: Mapping[str, NestedTensors],
264 hf_processor_mm_kwargs: Mapping[str, object],
265 ) -> Mapping[str, MultiModalFieldConfig]:
252 - return dict(images=MultiModalFieldConfig.batched("image"))
266 + return dict(
267 + images=MultiModalFieldConfig.batched("image"),
268 + embed_is_patch=MultiModalFieldConfig.batched("image"),
269 + )
270
271 def _get_prompt_updates(
272 self,
@@ -273,7 +290,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]
290 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
291 tokens[-1] = image_end_id
292
276 - return PromptUpdateDetails.select_token_id(tokens, image_token_id)
293 + return tokens
294
295 return [
296 PromptReplacement(
@@ -364,9 +381,17 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
381 raise ValueError("Incorrect type of images. "
382 f"Got type: {type(images)}")
383
384 + embed_is_patch = kwargs.pop("embed_is_patch")
385 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
386 + raise ValueError("Incorrect type of embed_is_patch. "
387 + f"Got type: {type(embed_is_patch)}")
388 +
389 + embed_is_patch = flatten_bn(embed_is_patch)
390 +
391 return PixtralImagePixelInputs(
392 type="pixel_values",
393 images=flatten_bn(images),
394 + embed_is_patch=embed_is_patch,
395 )
396
397 def _process_image_input(
@@ -402,7 +427,12 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
427 if image_input is None:
428 return None
429
405 - return self._process_image_input(image_input)
430 + image_features = self._process_image_input(image_input)
431 +
432 + return scatter_patch_features(
433 + image_features,
434 + image_input["embed_is_patch"],
435 + )
436
437 def get_input_embeddings(
438 self,
@@ -414,7 +444,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
444 inputs_embeds = merge_multimodal_embeddings(
445 input_ids,
446 inputs_embeds,
417 - multimodal_embeddings,
447 + select_patch_features(multimodal_embeddings),
448 self.vision_args.image_token_id,
449 )
450 return inputs_embeds
@@ -933,7 +963,9 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
963 image_width=image_width,
964 image_height=image_height,
965 )
936 - return ncols * nrows
966 +
967 + # Consider the image_break_token
968 + return (ncols + 1) * nrows
969
970 def get_max_image_tokens(self) -> int:
971 image_size = self.get_image_size()
vllm/model_executor/models/qwen2_audio.py
+3 -3
@@ -229,9 +229,9 @@ class Qwen2AudioMultiModalProcessor(
229
230 audio_tokens = [audio_token_id] * num_features
231
232 - return PromptUpdateDetails.select_token_id(
233 - [audio_bos_id] + audio_tokens + [audio_eos_id],
234 - embed_token_id=audio_token_id,
232 + return PromptUpdateDetails(
233 + full=[audio_bos_id] + audio_tokens + [audio_eos_id],
234 + features=audio_tokens,
235 )
236
237 return [
vllm/model_executor/models/qwen_vl.py
+3 -3
@@ -647,9 +647,9 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
647 PromptReplacement(
648 modality="image",
649 target=[img_start_id, img_end_id],
650 - replacement=PromptUpdateDetails.select_token_id(
651 - [img_start_id] + image_tokens + [img_end_id],
652 - embed_token_id=img_pad_id,
650 + replacement=PromptUpdateDetails(
651 + full=[img_start_id] + image_tokens + [img_end_id],
652 + features=image_tokens,
653 ),
654 )
655 ]
vllm/model_executor/models/skyworkr1v.py
+39 -3
@@ -40,6 +40,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
40 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
41 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
42 maybe_prefix, merge_multimodal_embeddings)
43 +from .vision import scatter_patch_features, select_patch_features
44
45 IMG_START = '<img>'
46 IMG_END = '</img>'
@@ -60,6 +61,14 @@ class SkyworkR1VImagePixelInputs(TypedDict):
61 num_patches: torch.Tensor
62 """Shape: `(batch_size * num_images)`"""
63
64 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
65 + """
66 + A boolean mask indicating which image embeddings correspond
67 + to patch tokens.
68 +
69 + Shape: `(batch_size * num_images, num_embeds)`
70 + """
71 +
72
73 class SkyworkR1VImageEmbeddingInputs(TypedDict):
74 type: Literal["image_embeds"]
@@ -410,13 +419,24 @@ class BaseSkyworkR1VProcessor(ABC):
419 torch.tensor([len(item) for item in pixel_values_lst]),
420 }
421
422 + tokenizer = self.tokenizer
423 + image_token_id = self.image_token_id
424 +
425 + embed_is_patch = list[torch.Tensor]()
426 +
427 for pixel_values in pixel_values_lst:
428 num_patches = pixel_values.shape[0]
429 feature_size = num_patches * self.num_image_token
430
431 image_repl = self.get_image_repl(feature_size, num_patches)
432 + feature_tokens = tokenizer.encode(image_repl.features,
433 + add_special_tokens=False)
434
435 text = [t.replace('<image>', image_repl.full, 1) for t in text]
436 + embed_is_patch.append(
437 + torch.tensor(feature_tokens) == image_token_id)
438 +
439 + image_inputs["embed_is_patch"] = embed_is_patch
440
441 text_inputs = self.tokenizer(text)
442
@@ -440,7 +460,7 @@ class SkyworkR1VProcessor(BaseSkyworkR1VProcessor):
460 repl_features = IMG_CONTEXT * feature_size
461 repl_full = IMG_START + repl_features + IMG_END
462
443 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
463 + return PromptUpdateDetails(full=repl_full, features=repl_features)
464
465
466 class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
@@ -579,6 +599,7 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
599 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
600 "image", image_num_patches),
601 image_num_patches=MultiModalFieldConfig.batched("image"),
602 + embed_is_patch=MultiModalFieldConfig.batched("image"),
603 image_embeds=MultiModalFieldConfig.batched("image"),
604 image_token_id=MultiModalFieldConfig.shared("image", num_images),
605 )
@@ -814,6 +835,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
835 self, **kwargs: object) -> Optional[SkyworkR1VImageInputs]:
836 pixel_values_flat = kwargs.pop("pixel_values_flat", None)
837 image_num_patches = kwargs.pop("image_num_patches", None)
838 + embed_is_patch = kwargs.pop("embed_is_patch", None)
839 image_embeds = kwargs.pop("image_embeds", None)
840
841 if pixel_values_flat is None and image_embeds is None:
@@ -842,14 +864,20 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
864 raise ValueError("Incorrect type of image_num_patches. "
865 f"Got type: {type(image_num_patches)}")
866
867 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
868 + raise ValueError("Incorrect type of embed_is_patch. "
869 + f"Got type: {type(embed_is_patch)}")
870 +
871 pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
872 image_num_patches = flatten_bn(image_num_patches, concat=True)
873 + embed_is_patch = flatten_bn(embed_is_patch)
874
875 return SkyworkR1VImagePixelInputs(
876 type="pixel_values",
877 pixel_values_flat=self._validate_pixel_values(
878 pixel_values_flat),
879 num_patches=image_num_patches,
880 + embed_is_patch=embed_is_patch,
881 )
882
883 raise AssertionError("This line should be unreachable.")
@@ -895,7 +923,15 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
923 if image_input is None:
924 return None
925
898 - return self._process_image_input(image_input)
926 + image_features = self._process_image_input(image_input)
927 +
928 + if image_input["type"] != "pixel_values":
929 + return image_features
930 +
931 + return scatter_patch_features(
932 + image_features,
933 + image_input["embed_is_patch"],
934 + )
935
936 def get_input_embeddings(
937 self,
@@ -909,7 +945,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
945 inputs_embeds = merge_multimodal_embeddings(
946 input_ids,
947 inputs_embeds,
912 - multimodal_embeddings,
948 + select_patch_features(multimodal_embeddings),
949 self.img_context_token_id,
950 )
951 return inputs_embeds
vllm/model_executor/models/vision.py
+76 -1
@@ -1,7 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from abc import ABC, abstractmethod
4 -from typing import Final, Generic, Optional, Protocol, TypeVar, Union
4 +from collections.abc import Sequence
5 +from typing import Final, Generic, Optional, Protocol, TypeVar, Union, cast
6
7 import torch
8 from transformers import PretrainedConfig
@@ -9,9 +10,12 @@ from transformers import PretrainedConfig
10 import vllm.envs as envs
11 from vllm.attention.selector import (backend_name_to_enum,
12 get_global_forced_attn_backend)
13 +from vllm.jsontree import JSONTree, json_map_leaves
14 from vllm.logger import init_logger
15 from vllm.platforms import _Backend, current_platform
16
17 +from .interfaces import MultiModalEmbeddings
18 +
19 logger = init_logger(__name__)
20
21 _C = TypeVar("_C", bound=PretrainedConfig)
@@ -151,3 +155,74 @@ def resolve_visual_encoder_outputs(
155 if post_layer_norm is not None and uses_last_layer:
156 hs_pool[-1] = post_layer_norm(encoder_outputs)
157 return torch.cat(hs_pool, dim=-1)
158 +
159 +
160 +def scatter_patch_features(
161 + patches: Union[torch.Tensor, Sequence[torch.Tensor]],
162 + embed_is_patch: Union[torch.Tensor, Sequence[torch.Tensor]],
163 +) -> tuple[torch.Tensor, ...]:
164 + """
165 + Scatter the patch features into a contiguous tensor that corresponds
166 + to the embedding tokens defined by the multimodal processor.
167 +
168 + The rest of the values in the tensor are set to NaN so that they
169 + can be filtered out by :func`select_patch_features`.
170 +
171 + Args:
172 + patches: The patch features for each image.
173 + Shape: `(num_images, <patch_dims>, feature_depth)`
174 + embed_is_patch: A boolean mask indicating which image embeddings
175 + correspond to patch tokens for each image.
176 + Shape: `(num_images, num_embeds)`
177 +
178 + Note:
179 + The original code only considers patch tokens as feature
180 + tokens, but our processor considers all image-related tokens
181 + as feature tokens because the feature tokens need to be
182 + consecutive in `input_ids`.
183 +
184 + Example:
185 + A simplified example for one image:
186 +
187 + .. code-block::
188 +
189 + Embedding tokens (from HF processor):
190 + [<start> <patch> <patch> <col> <patch> <patch> <col> <end> ]
191 +
192 + embed_is_patch (from HF processor):
193 + [ False True True False True True False False ]
194 +
195 + Encoder outputs (from model):
196 + [ p1 p2 p3 p4 ]
197 +
198 + The resulting embedding tensor is:
199 + [ nan p1 p2 nan p3 p4 nan nan ]
200 + """
201 + if len(patches) != len(embed_is_patch):
202 + raise ValueError(f"Inconsistent num_images: {len(patches)=} vs. "
203 + f"{len(embed_is_patch)=}")
204 +
205 + def get_embed_one(patches_one: torch.Tensor, e_is_patch: torch.Tensor):
206 + embed_one = patches_one.new_full(
207 + (e_is_patch.shape[0], patches_one.shape[-1]),
208 + fill_value=torch.nan,
209 + )
210 + embed_one[e_is_patch] = patches_one
211 + return embed_one
212 +
213 + return tuple(
214 + get_embed_one(patches_one, e_is_patch)
215 + for patches_one, e_is_patch in zip(patches, embed_is_patch))
216 +
217 +
218 +def select_patch_features(
219 + multimodal_embeddings: MultiModalEmbeddings) -> MultiModalEmbeddings:
220 + """
221 + Given the outputs of :func:`scatter_patch_features`, return only
222 + the values that correspond to patch features.
223 + """
224 + selected_features = json_map_leaves(
225 + lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),
226 + cast(JSONTree[torch.Tensor], multimodal_embeddings),
227 + )
228 + return cast(MultiModalEmbeddings, selected_features)
vllm/multimodal/base.py
+2 -2
@@ -385,8 +385,8 @@ class MultiModalPlaceholderMap:
385 for placeholder_dict, mm_item in zip(multi_modal_placeholders,
386 multi_modal_items):
387 placeholder = range(
388 - placeholder_dict.offset,
389 - placeholder_dict.offset + placeholder_dict.length,
388 + placeholder_dict["offset"],
389 + placeholder_dict["offset"] + placeholder_dict["length"],
390 )
391 intersection = range(
392 max(positions.start, placeholder.start),
vllm/multimodal/inputs.py
+3 -29
@@ -109,8 +109,7 @@ The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
109 """
110
111
112 -@dataclass(frozen=True)
113 -class PlaceholderRange:
112 +class PlaceholderRange(TypedDict):
113 """
114 Placeholder location information for multi-modal data.
115
@@ -122,8 +121,8 @@ class PlaceholderRange:
121
122 .. code-block::
123
125 - A: PlaceholderRange(offset=0, length=4)
126 - B: PlaceholderRange(offset=5, length=4)
124 + A: { "offset": 0, "length": 4 }
125 + B: { "offset": 5, "length": 4 }
126 """
127
128 offset: int
@@ -132,31 +131,6 @@ class PlaceholderRange:
131 length: int
132 """The length of the placeholder."""
133
135 - is_embed: Optional[torch.Tensor] = None
136 - """
137 - A boolean mask of shape `(length,)` indicating which positions
138 - between `offset` and `offset + length` to assign embeddings to.
139 - """
140 -
141 - def get_num_embeds(self) -> int:
142 - if self.is_embed is None:
143 - return self.length
144 -
145 - return int(self.is_embed.sum().item())
146 -
147 - def __eq__(self, other: object) -> bool:
148 - if not isinstance(other, self.__class__):
149 - return False
150 - if not (self.offset, self.length) == (other.offset, other.length):
151 - return False
152 -
153 - if self.is_embed is None:
154 - return other.is_embed is None
155 - if other.is_embed is None:
156 - return self.is_embed is None
157 -
158 - return nested_tensors_equal(self.is_embed, other.is_embed)
159 -
134
135 NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
136 tuple[torch.Tensor, ...]]
vllm/multimodal/processing.py
+25 -52
@@ -108,46 +108,16 @@ class PromptUpdateDetails(Generic[_S]):
108 full: _S
109 """The full content."""
110
111 - is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None
111 + features: _S
112 """
113 - Given :attr:`full`, return a boolean mask of shape `(len(full),)`
114 - indicating which positions of `full` to assign embeddings to.
115 -
116 - `None` (default) means to assign embeddings to all positions of `full`.
117 -
118 - The embeddings are obtained by calling
119 - :class:`SupportsMultiModal.get_multimodal_embeddings`.
113 + The part of the content that corresponds to feature placeholders;
114 + this will be replaced by the output of the vision encoder during model
115 + inference.
116 """
117
118 @staticmethod
119 def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
124 - return PromptUpdateDetails(full=seq)
125 -
126 - @staticmethod
127 - def select_text(
128 - seq: _S,
129 - embed_text: str,
130 - ) -> "PromptUpdateDetails[_S]":
131 -
132 - def is_embed(full: "_BoundPromptSequence") -> torch.Tensor:
133 - embed_token_ids = encode_tokens(full.tokenizer, embed_text)
134 -
135 - return torch.isin(
136 - torch.tensor(full.token_ids),
137 - torch.tensor(embed_token_ids),
138 - )
139 -
140 - return PromptUpdateDetails(full=seq, is_embed=is_embed)
141 -
142 - @staticmethod
143 - def select_token_id(
144 - seq: _S,
145 - embed_token_id: int,
146 - ) -> "PromptUpdateDetails[_S]":
147 - return PromptUpdateDetails(
148 - full=seq,
149 - is_embed=lambda f: torch.tensor(f.token_ids) == embed_token_id,
150 - )
120 + return PromptUpdateDetails(full=seq, features=seq)
121
122
123 PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
@@ -436,7 +406,7 @@ class _BoundPromptSequence:
406 @dataclass
407 class _BoundPromptContent:
408 full: _BoundPromptSequence
439 - is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]]
409 + features: _BoundPromptSequence
410
411
412 @dataclass
@@ -496,8 +466,10 @@ class BoundPromptUpdate:
466
467 bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
468 content.full)
469 + bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
470 + content.features)
471 bound_content = _BoundPromptContent(full=bound_full,
500 - is_embed=content.is_embed)
472 + features=bound_features)
473
474 if cache_key is not None:
475 self._content_cache[cache_key] = bound_content
@@ -633,19 +605,15 @@ class PlaceholderFeaturesInfo:
605 item_idx: int
606 start_idx: int
607 tokens: list[int]
636 - is_embed: Optional[torch.Tensor]
608
609 @property
610 def length(self) -> int:
611 return len(self.tokens)
612
613 def to_range(self) -> PlaceholderRange:
643 - # TODO: Is it worth it to optimize this by stripping the
644 - # leading and ending positions where `is_embed=False`?
614 return PlaceholderRange(
615 offset=self.start_idx,
616 length=self.length,
648 - is_embed=self.is_embed,
617 )
618
619
@@ -838,17 +806,22 @@ def _iter_placeholders(
806 continue
807
808 if prompt[start_idx:end_idx_full] == content_tokens_full:
841 - content_is_embed = content.is_embed
842 - if content_is_embed is not None:
843 - content_is_embed = content_is_embed(content.full)
844 -
845 - yield PlaceholderFeaturesInfo(
846 - modality=modality,
847 - item_idx=item_idx,
848 - start_idx=start_idx,
849 - tokens=content_tokens_full,
850 - is_embed=content_is_embed,
851 - )
809 + content_tokens_feat = content.features.token_ids
810 +
811 + try:
812 + match = next(
813 + iter_token_matches(content_tokens_full,
814 + content_tokens_feat))
815 + yield PlaceholderFeaturesInfo(
816 + modality=modality,
817 + item_idx=item_idx,
818 + start_idx=start_idx + match.start_idx,
819 + tokens=content_tokens_feat,
820 + )
821 + except StopIteration:
822 + raise AssertionError(
823 + f"{content_tokens_feat=} should be a "
824 + f"subsequence of {content_tokens_full=}") from None
825
826 # Exclude overlapping matches
827 start_idx = end_idx_full
vllm/multimodal/profiling.py
+1 -1
@@ -180,7 +180,7 @@ class MultiModalProfiler(Generic[_I]):
180 placeholders_by_modality = mm_inputs["mm_placeholders"]
181
182 total_placeholders_by_modality = {
183 - modality: sum(item.get_num_embeds() for item in placeholders)
183 + modality: sum(item["length"] for item in placeholders)
184 for modality, placeholders in placeholders_by_modality.items()
185 }
186 expected_placeholders_by_modality = {
vllm/multimodal/utils.py
+1 -1
@@ -340,7 +340,7 @@ def merge_and_sort_multimodal_metadata(
340 all_items.append((modality, placeholder, hash_value))
341
342 # Sort all items by offset
343 - all_items.sort(key=lambda x: x[1].offset)
343 + all_items.sort(key=lambda x: x[1]['offset'])
344
345 # Split into separate lists
346 sorted_modalities = [item[0] for item in all_items]
vllm/v1/core/kv_cache_utils.py
+4 -3
@@ -310,7 +310,8 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
310 # Note that we assume mm_positions is sorted by offset.
311 # We do not need to check all mm inputs if the start token index is out of
312 # range. This usually happens in the late prefill phase and decoding phase.
313 - if mm_positions[-1].offset + mm_positions[-1].length < start_token_idx:
313 + if mm_positions[-1]["offset"] + mm_positions[-1][
314 + "length"] < start_token_idx:
315 return extra_keys, start_mm_idx
316
317 # Support start_mm_idx == -1 to indicate the last mm input.
@@ -321,8 +322,8 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
322 curr_mm_idx = start_mm_idx
323 while mm_positions and curr_mm_idx < len(mm_positions):
324 assert mm_hashes[curr_mm_idx] is not None
324 - offset = mm_positions[curr_mm_idx].offset
325 - length = mm_positions[curr_mm_idx].length
325 + offset = mm_positions[curr_mm_idx]["offset"]
326 + length = mm_positions[curr_mm_idx]["length"]
327 if end_token_idx > offset:
328 if start_token_idx > offset + length:
329 # This block has passed the current mm input.
vllm/v1/core/sched/scheduler.py
+4 -4
@@ -505,8 +505,8 @@ class Scheduler(SchedulerInterface):
505 assert mm_positions is not None
506 assert len(mm_positions) > 0
507 for i, pos_info in enumerate(mm_positions):
508 - start_pos = pos_info.offset
509 - num_encoder_tokens = pos_info.length
508 + start_pos = pos_info["offset"]
509 + num_encoder_tokens = pos_info["length"]
510
511 # The encoder output is needed if the two ranges overlap:
512 # [num_computed_tokens, num_computed_tokens + num_new_tokens) and
@@ -596,8 +596,8 @@ class Scheduler(SchedulerInterface):
596 if cached_encoder_input_ids:
597 for input_id in list(cached_encoder_input_ids):
598 mm_positions = request.mm_positions[input_id]
599 - start_pos = mm_positions.offset
600 - num_tokens = mm_positions.length
599 + start_pos = mm_positions["offset"]
600 + num_tokens = mm_positions["length"]
601 if start_pos + num_tokens <= request.num_computed_tokens:
602 # The encoder output is already processed and stored
603 # in the decoder's KV cache.
vllm/v1/request.py
+1 -1
@@ -121,7 +121,7 @@ class Request:
121
122 def get_num_encoder_tokens(self, input_id: int) -> int:
123 assert input_id < len(self.mm_positions)
124 - num_tokens = self.mm_positions[input_id].length
124 + num_tokens = self.mm_positions[input_id]["length"]
125 return num_tokens
126
127 @property
vllm/v1/worker/gpu_model_runner.py
+20 -40
@@ -19,8 +19,7 @@ from vllm.logger import init_logger
19 from vllm.model_executor.layers.fused_moe import FusedMoE
20 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
21 from vllm.model_executor.model_loader import get_model
22 -from vllm.multimodal import MULTIMODAL_REGISTRY
23 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
22 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
23 from vllm.multimodal.utils import group_mm_inputs_by_modality
24 from vllm.sampling_params import SamplingType
25 from vllm.sequence import IntermediateTensors
@@ -44,8 +43,7 @@ from vllm.v1.utils import bind_kv_cache
43 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
44 from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
45
47 -from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,
48 - scatter_mm_placeholders)
46 +from .utils import sanity_check_mm_encoder_outputs
47
48 if TYPE_CHECKING:
49 import xgrammar as xgr
@@ -831,22 +829,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
829 )
830 return metadata
831
834 - def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
832 + def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
833 scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
834 if not scheduled_encoder_inputs:
835 return
836
837 # Batch the multi-modal inputs.
840 - mm_inputs = list[MultiModalKwargs]()
841 - req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
838 + mm_inputs: list[MultiModalKwargs] = []
839 + req_input_ids: list[tuple[str, int]] = []
840 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
841 req_state = self.requests[req_id]
844 - for input_id, pos_info in zip(
845 - encoder_input_ids,
846 - req_state.mm_positions,
847 - ):
842 + for input_id in encoder_input_ids:
843 mm_inputs.append(req_state.mm_inputs[input_id])
849 - req_ids_pos.append((req_id, input_id, pos_info))
844 + req_input_ids.append((req_id, input_id))
845
846 # Batch mm inputs as much as we can: if a request in the batch has
847 # multiple modalities or a different modality than the previous one,
@@ -882,23 +877,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
877 encoder_outputs.append(output)
878
879 # Cache the encoder outputs.
885 - for (req_id, input_id, pos_info), output in zip(
886 - req_ids_pos,
887 - encoder_outputs,
888 - ):
880 + for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
881 if req_id not in self.encoder_cache:
882 self.encoder_cache[req_id] = {}
883 + self.encoder_cache[req_id][input_id] = output
884
892 - self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(
893 - output,
894 - is_embed=pos_info.is_embed,
895 - )
896 -
897 - def _gather_mm_embeddings(
885 + def _gather_encoder_outputs(
886 self,
887 scheduler_output: "SchedulerOutput",
888 ) -> list[torch.Tensor]:
901 - mm_embeds: list[torch.Tensor] = []
889 + encoder_outputs: list[torch.Tensor] = []
890 for req_id in self.input_batch.req_ids:
891 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
892 req_id]
@@ -906,8 +894,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
894 num_computed_tokens = req_state.num_computed_tokens
895 mm_positions = req_state.mm_positions
896 for i, pos_info in enumerate(mm_positions):
909 - start_pos = pos_info.offset
910 - num_encoder_tokens = pos_info.length
897 + start_pos = pos_info["offset"]
898 + num_encoder_tokens = pos_info["length"]
899
900 # The encoder output is needed if the two ranges overlap:
901 # [num_computed_tokens,
@@ -929,16 +917,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
917 assert req_id in self.encoder_cache
918 assert i in self.encoder_cache[req_id]
919 encoder_output = self.encoder_cache[req_id][i]
932 -
933 - if (is_embed := pos_info.is_embed) is not None:
934 - is_embed = is_embed[start_idx:end_idx]
935 -
936 - mm_embeds_item = gather_mm_placeholders(
937 - encoder_output[start_idx:end_idx],
938 - is_embed=is_embed,
939 - )
940 - mm_embeds.append(mm_embeds_item)
941 - return mm_embeds
920 + encoder_outputs.append(encoder_output[start_idx:end_idx])
921 + return encoder_outputs
922
923 def get_model(self) -> nn.Module:
924 return self.model
@@ -1003,10 +983,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
983
984 if self.is_multimodal_model:
985 # Run the multimodal encoder if any.
1006 - self._execute_mm_encoder(scheduler_output)
1007 - mm_embeds = self._gather_mm_embeddings(scheduler_output)
986 + self._execute_encoder(scheduler_output)
987 + encoder_outputs = self._gather_encoder_outputs(scheduler_output)
988 else:
1009 - mm_embeds = []
989 + encoder_outputs = []
990
991 # Prepare the decoder inputs.
992 attn_metadata, logits_indices, spec_decode_metadata = (
@@ -1028,9 +1008,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1008 # embeddings), we always use embeddings (rather than token ids)
1009 # as input to the multimodal model, even when the input is text.
1010 input_ids = self.input_ids[:num_scheduled_tokens]
1031 - if mm_embeds:
1011 + if encoder_outputs:
1012 inputs_embeds = self.model.get_input_embeddings(
1033 - input_ids, mm_embeds)
1013 + input_ids, encoder_outputs)
1014 else:
1015 inputs_embeds = self.model.get_input_embeddings(input_ids)
1016 # TODO(woosuk): Avoid the copy. Optimize.
vllm/v1/worker/tpu_model_runner.py
+20 -65
@@ -19,8 +19,7 @@ from vllm.config import VllmConfig
19 from vllm.forward_context import set_forward_context
20 from vllm.logger import init_logger
21 from vllm.model_executor.model_loader import get_model
22 -from vllm.multimodal import MULTIMODAL_REGISTRY
23 -from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange
22 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
23 from vllm.multimodal.utils import group_mm_inputs_by_modality
24 from vllm.sampling_params import SamplingType
25 from vllm.sequence import IntermediateTensors
@@ -37,8 +36,7 @@ from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler
36 from vllm.v1.utils import bind_kv_cache
37 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
38
40 -from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,
41 - scatter_mm_placeholders)
39 +from .utils import sanity_check_mm_encoder_outputs
40
41 if TYPE_CHECKING:
42 from vllm.v1.core.sched.output import SchedulerOutput
@@ -509,47 +507,19 @@ class TPUModelRunner:
507 logits_indices = logits_indices.to(self.device)
508 return attn_metadata, logits_indices
509
512 - def _scatter_placeholders(
513 - self,
514 - embeds: torch.Tensor,
515 - is_embed: Optional[torch.Tensor],
516 - ) -> torch.Tensor:
517 - if is_embed is None:
518 - return embeds
519 -
520 - placeholders = embeds.new_full(
521 - (is_embed.shape[0], embeds.shape[-1]),
522 - fill_value=torch.nan,
523 - )
524 - placeholders[is_embed] = embeds
525 - return placeholders
526 -
527 - def _gather_placeholders(
528 - self,
529 - placeholders: torch.Tensor,
530 - is_embed: Optional[torch.Tensor],
531 - ) -> torch.Tensor:
532 - if is_embed is None:
533 - return placeholders
534 -
535 - return placeholders[is_embed]
536 -
537 - def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):
510 + def _execute_encoder(self, scheduler_output: "SchedulerOutput"):
511 scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
512 if not scheduled_encoder_inputs:
513 return
514
515 # Batch the multi-modal inputs.
543 - mm_inputs = list[MultiModalKwargs]()
544 - req_ids_pos = list[tuple[str, int, PlaceholderRange]]()
516 + mm_inputs: list[MultiModalKwargs] = []
517 + req_input_ids: list[tuple[str, int]] = []
518 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
519 req_state = self.requests[req_id]
547 - for input_id, pos_info in zip(
548 - encoder_input_ids,
549 - req_state.mm_positions,
550 - ):
520 + for input_id in encoder_input_ids:
521 mm_inputs.append(req_state.mm_inputs[input_id])
552 - req_ids_pos.append((req_id, input_id, pos_info))
522 + req_input_ids.append((req_id, input_id))
523
524 # Batch mm inputs as much as we can: if a request in the batch has
525 # multiple modalities or a different modality than the previous one,
@@ -585,23 +555,16 @@ class TPUModelRunner:
555 encoder_outputs.append(output)
556
557 # Cache the encoder outputs.
588 - for (req_id, input_id, pos_info), output in zip(
589 - req_ids_pos,
590 - encoder_outputs,
591 - ):
558 + for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):
559 if req_id not in self.encoder_cache:
560 self.encoder_cache[req_id] = {}
561 + self.encoder_cache[req_id][input_id] = output
562
595 - self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(
596 - output,
597 - is_embed=pos_info.is_embed,
598 - )
599 -
600 - def _gather_mm_embeddings(
563 + def _gather_encoder_outputs(
564 self,
565 scheduler_output: "SchedulerOutput",
566 ) -> list[torch.Tensor]:
604 - mm_embeds: list[torch.Tensor] = []
567 + encoder_outputs: list[torch.Tensor] = []
568 for req_id in self.input_batch.req_ids:
569 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
570 req_id]
@@ -609,8 +572,8 @@ class TPUModelRunner:
572 num_computed_tokens = req_state.num_computed_tokens
573 mm_positions = req_state.mm_positions
574 for i, pos_info in enumerate(mm_positions):
612 - start_pos = pos_info.offset
613 - num_encoder_tokens = pos_info.length
575 + start_pos = pos_info["offset"]
576 + num_encoder_tokens = pos_info["length"]
577
578 # The encoder output is needed if the two ranges overlap:
579 # [num_computed_tokens,
@@ -632,16 +595,8 @@ class TPUModelRunner:
595 assert req_id in self.encoder_cache
596 assert i in self.encoder_cache[req_id]
597 encoder_output = self.encoder_cache[req_id][i]
635 -
636 - if (is_embed := pos_info.is_embed) is not None:
637 - is_embed = is_embed[start_idx:end_idx]
638 -
639 - mm_embeds_item = gather_mm_placeholders(
640 - encoder_output[start_idx:end_idx],
641 - is_embed=is_embed,
642 - )
643 - mm_embeds.append(mm_embeds_item)
644 - return mm_embeds
598 + encoder_outputs.append(encoder_output[start_idx:end_idx])
599 + return encoder_outputs
600
601 @torch.no_grad()
602 def execute_model(
@@ -657,10 +612,10 @@ class TPUModelRunner:
612
613 if self.is_multimodal_model:
614 # Run the multimodal encoder if any.
660 - self._execute_mm_encoder(scheduler_output)
661 - mm_embeds = self._gather_mm_embeddings(scheduler_output)
615 + self._execute_encoder(scheduler_output)
616 + encoder_outputs = self._gather_encoder_outputs(scheduler_output)
617 else:
663 - mm_embeds = []
618 + encoder_outputs = []
619
620 # Prepare inputs
621 attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)
@@ -668,9 +623,9 @@ class TPUModelRunner:
623 # NOTE(woosuk): To unify token ids and soft tokens (vision
624 # embeddings), we always use embeddings (rather than token ids)
625 # as input to the multimodal model, even when the input is text.
671 - if mm_embeds:
626 + if encoder_outputs:
627 inputs_embeds = self.model.get_input_embeddings(
673 - self.input_ids, mm_embeds)
628 + self.input_ids, encoder_outputs)
629 else:
630 inputs_embeds = self.model.get_input_embeddings(self.input_ids)
631 input_ids = None
vllm/v1/worker/utils.py
-45
@@ -1,6 +1,4 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -from typing import Optional
3 -
2 import torch
3
4
@@ -29,46 +27,3 @@ def sanity_check_mm_encoder_outputs(
27 f"but got tensors with shapes {[e.shape for e in mm_embeddings]} "
28 "instead. This is most likely due to incorrect implementation "
29 "of the model's `get_multimodal_embeddings` method.")
32 -
33 -
34 -def scatter_mm_placeholders(
35 - embeds: torch.Tensor,
36 - is_embed: Optional[torch.Tensor],
37 -) -> torch.Tensor:
38 - """
39 - Scatter the multimodal embeddings into a contiguous tensor that represents
40 - the placeholder tokens.
41 -
42 - :class:`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
43 -
44 - Args:
45 - embeds: The multimodal embeddings.
46 - Shape: `(num_embeds, embed_dim)`
47 - is_embed: A boolean mask indicating which positions in the placeholder
48 - tokens need to be filled with multimodal embeddings.
49 - Shape: `(num_placeholders, num_embeds)`
50 - """
51 - if is_embed is None:
52 - return embeds
53 -
54 - placeholders = embeds.new_full(
55 - (is_embed.shape[0], embeds.shape[-1]),
56 - fill_value=torch.nan,
57 - )
58 - placeholders[is_embed] = embeds
59 - return placeholders
60 -
61 -
62 -def gather_mm_placeholders(
63 - placeholders: torch.Tensor,
64 - is_embed: Optional[torch.Tensor],
65 -) -> torch.Tensor:
66 - """
67 - Reconstructs the embeddings from the placeholder tokens.
68 -
69 - This is the operation of :func:`scatter_mm_placeholders`.
70 - """
71 - if is_embed is None:
72 - return placeholders
73 -
74 - return placeholders[is_embed]