@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f2ebb6f541

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

[V1] Scatter and gather placeholders in the model runner (#16076)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Jennifer Zhao <ai.jenniferzhao@gmail.com>

Roger Wang committed Apr 7, 2025 at 19:43 UTC f2ebb6f541ab6afa30907232df6883319a0a4d83

41 files changed +522 -1021

docs/source/contributing/model/multimodal.md

+8 -8

@@ -860,8 +860,8 @@ prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch(

860 )

861 ```

862

863 -To accommodate this, instead of a string you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`

864 -with different `full` and `feature` attributes:

863 +To assign the vision embeddings to only the image tokens, instead of a string

864 +you can return an instance of {class}`~vllm.multimodal.processing.PromptUpdateDetails`:

865

866 ```python

867 hf_config = self.info.get_hf_config()

@@ -879,9 +879,9 @@ def get_replacement_fuyu(item_idx: int):

879 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +

880 [_NEWLINE_TOKEN_ID]) * nrows

881

882 - return PromptUpdateDetails(

883 - full=image_tokens + [bos_token_id],

884 - features=image_tokens,

882 + return PromptUpdateDetails.select_token_id(

883 + image_tokens + [bos_token_id],

884 + embed_token_id=_IMAGE_TOKEN_ID,

885 )

886 ```

887

@@ -914,9 +914,9 @@ def _get_prompt_updates(

914 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +

915 [_NEWLINE_TOKEN_ID]) * nrows

916

917 - return PromptUpdateDetails(

918 - full=image_tokens + [bos_token_id],

919 - features=image_tokens,

917 + return PromptUpdateDetails.select_token_id(

918 + image_tokens + [bos_token_id],

919 + embed_token_id=_IMAGE_TOKEN_ID,

920 )

921

922 return [

docs/source/models/supported_models.md

-3

@@ -1006,9 +1006,6 @@ See [this page](#generative-models) for more information on how to use generativ

1006 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.

1007

1008 :::{important}

1009 -To use Gemma3 series models, you have to install Hugging Face Transformers library from source via

1010 -`pip install git+https://github.com/huggingface/transformers`.

1011 -

1009 Pan-and-scan image pre-processing is currently supported on V0 (but not V1).

1010 You can enable it by passing `--mm-processor-kwargs '{"do_pan_and_scan": True}'`.

1011 :::

tests/models/decoder_only/vision_language/test_models.py

+2 -3

@@ -330,9 +330,8 @@ VLM_TEST_SETTINGS = {

330 max_num_seqs=4,

331 dtype="bfloat16",

332 auto_cls=AutoModelForImageTextToText,

333 - tensor_parallel_size=8,

334 - vllm_runner_kwargs={"gpu_memory_utilization": 0.8},

335 - marks=multi_gpu_marks(num_gpus=8),

333 + tensor_parallel_size=4,

334 + marks=multi_gpu_marks(num_gpus=4),

335 ),

336 "llava_next": VLMTestInfo(

337 models=["llava-hf/llava-v1.6-mistral-7b-hf"],

tests/models/decoder_only/vision_language/test_pixtral.py

+8 -16

@@ -200,22 +200,14 @@ def test_chat(

200

201

202 @large_gpu_test(min_gb=48)

203 -@pytest.mark.parametrize(

204 - "prompt,expected_ranges",

205 - [(_create_engine_inputs_hf(IMG_URLS[:1]), [{

206 - "offset": 11,

207 - "length": 494

208 - }]),

209 - (_create_engine_inputs_hf(IMG_URLS[1:4]), [{

210 - "offset": 11,

211 - "length": 266

212 - }, {

213 - "offset": 277,

214 - "length": 1056

215 - }, {

216 - "offset": 1333,

217 - "length": 418

218 - }])])

203 +@pytest.mark.parametrize("prompt,expected_ranges",

204 + [(_create_engine_inputs_hf(IMG_URLS[:1]),

205 + [PlaceholderRange(offset=11, length=494)]),

206 + (_create_engine_inputs_hf(IMG_URLS[1:4]), [

207 + PlaceholderRange(offset=11, length=266),

208 + PlaceholderRange(offset=277, length=1056),

209 + PlaceholderRange(offset=1333, length=418)

210 + ])])

211 def test_multi_modal_placeholders(vllm_runner, prompt,

212 expected_ranges: list[PlaceholderRange],

213 monkeypatch) -> None:

tests/models/multimodal/processing/test_llama4.py

+1 -11

@@ -71,13 +71,11 @@ def test_processor_override(

71 # image token offsets

72 img_locs = processed_inputs["mm_placeholders"].get("image", [])

73 assert len(img_locs) == num_imgs

74 - assert [img_loc["offset"] for img_loc in img_locs] == \

74 + assert [img_loc.offset for img_loc in img_locs] == \

75 [i for i, v in enumerate(prompt_token_ids) \

76 if v == config.boi_token_index]

78 # patch sizes and masks

79 - assert prompt_token_ids.count(config.image_token_index) \

80 - == sum(img_patch.sum() for img_patch in mm_kwargs["embed_is_patch"])

79 patch_token_id = vocab[hf_processor.img_patch_token]

80 num_patches = processed_inputs["prompt_token_ids"].count(patch_token_id)

81 mm_counts = {"image": num_imgs}

@@ -89,11 +87,3 @@ def test_processor_override(

87 == mm_kwargs["patches_per_image"].sum() * num_patches_per_chunk

88 assert mm_kwargs["pixel_values"].shape[0] \

89 == mm_kwargs["patches_per_image"].sum()

92 -

93 - for embed_is_patch, aspect_ratio in zip(mm_kwargs["embed_is_patch"],

94 - mm_kwargs["aspect_ratios"]):

95 - assert embed_is_patch.shape[0] == \

96 - len(tokenizer.encode(

97 - hf_processor._prompt_split_image(

98 - aspect_ratio, num_patches_per_chunk),

99 - add_special_tokens=False))

tests/models/multimodal/processing/test_llava_next.py

+2 -2

@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(

92 first_placeholder = image_placeholders[0]

94 # NOTE: There is a BOS token

95 - assert first_placeholder["offset"] == 1

96 - assert first_placeholder["length"] == (

95 + assert first_placeholder.offset == 1

96 + assert first_placeholder.length == (

97 len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs

99 except Exception as exc:

tests/models/multimodal/processing/test_llava_onevision.py

+2 -2

@@ -92,8 +92,8 @@ def _validate_image_prompt_replacements_one(

93 first_placeholder = image_placeholders[0]

95 - assert first_placeholder["offset"] == 0

96 - assert first_placeholder["length"] == len(

95 + assert first_placeholder.offset == 0

96 + assert first_placeholder.length == len(

97 processed_inputs["prompt_token_ids"]) // num_imgs

98 except Exception as exc:

99 failed_size_excs.append((image_size, exc))

tests/multimodal/test_processing.py

@@ -785,6 +785,7 @@ def test_find_update_tokens(

785 item_idx=0,

786 start_idx=6,

787 tokens=[32000, 32000],

788 + is_embed=None,

789 ),

790 ],

791 "pattern_4": [

@@ -793,6 +794,7 @@ def test_find_update_tokens(

794 item_idx=0,

795 start_idx=3,

796 tokens=[32000],

797 + is_embed=None,

798 ),

799 ],

800 }

@@ -807,12 +809,14 @@ def test_find_update_tokens(

809 item_idx=0,

810 start_idx=1,

811 tokens=[32000, 32000],

812 + is_embed=None,

813 ),

814 PlaceholderFeaturesInfo(

815 modality="pattern_1",

816 item_idx=1,

817 start_idx=5,

818 tokens=[32000, 32000],

819 + is_embed=None,

820 ),

821 ],

822 "pattern_3": [

@@ -821,6 +825,7 @@ def test_find_update_tokens(

825 item_idx=0,

826 start_idx=7,

827 tokens=[1550, 918, 1550],

828 + is_embed=None,

829 ),

830 ],

831 # No match for pattern_4 as it has lower priority than pattern_1

@@ -835,12 +840,14 @@ def test_find_update_tokens(

840 item_idx=0,

841 start_idx=1,

842 tokens=[32000, 32000],

843 + is_embed=None,

844 ),

845 PlaceholderFeaturesInfo(

846 modality="pattern_1",

847 item_idx=1,

848 start_idx=3,

849 tokens=[32000, 32000],

850 + is_embed=None,

851 ),

852 ],

853 "pattern_4": [

@@ -849,6 +856,7 @@ def test_find_update_tokens(

856 item_idx=0,

857 start_idx=5,

858 tokens=[32000],

859 + is_embed=None,

860 ),

861 ],

862 "pattern_3": [

@@ -857,6 +865,7 @@ def test_find_update_tokens(

865 item_idx=0,

866 start_idx=6,

867 tokens=[1550, 918, 1550],

868 + is_embed=None,

869 ),

870 ],

871 }

tests/v1/core/test_kv_cache_utils.py

+17 -29

@@ -3,7 +3,7 @@

3 import pytest

4 import torch

6 -from vllm.multimodal.inputs import MultiModalKwargs

6 +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange

7 from vllm.sampling_params import SamplingParams

8 from vllm.utils import sha256

9 # disable yapf here as it formats differently than isort such that both fail

@@ -158,13 +158,10 @@ def test_generate_block_hash_extra_keys():

158 request = make_request(

159 request_id=0,

160 prompt_token_ids=[_ for _ in range(20)],

161 - mm_positions=[{

162 - "offset": 0,

163 - "length": 5

164 - }, {

165 - "offset": 10,

166 - "length": 5

167 - }],

161 + mm_positions=[

162 + PlaceholderRange(offset=0, length=5),

163 + PlaceholderRange(offset=10, length=5),

164 + ],

165 mm_hashes=["hash1", "hash2"],

166 )

167

@@ -222,13 +219,10 @@ def test_hash_request_tokens(hash_fn):

219 request = make_request(

220 request_id=0,

221 prompt_token_ids=[_ for _ in range(6)],

225 - mm_positions=[{

226 - "offset": 0,

227 - "length": 3

228 - }, {

229 - "offset": 3,

230 - "length": 3

231 - }],

222 + mm_positions=[

223 + PlaceholderRange(offset=0, length=3),

224 + PlaceholderRange(offset=3, length=3),

225 + ],

226 mm_hashes=["hash1", "hash2"],

227 )

228

@@ -253,25 +247,19 @@ def test_hash_tokens_different_mm_input(hash_fn):

247 request1 = make_request(

248 request_id=0,

249 prompt_token_ids=[_ for _ in range(6)],

256 - mm_positions=[{

257 - "offset": 0,

258 - "length": 3

259 - }, {

260 - "offset": 3,

261 - "length": 3

262 - }],

250 + mm_positions=[

251 + PlaceholderRange(offset=0, length=3),

252 + PlaceholderRange(offset=3, length=3),

253 + ],

254 mm_hashes=["hash1", "hash2"],

255 )

256 request2 = make_request(

257 request_id=1,

258 prompt_token_ids=[_ for _ in range(6)],

268 - mm_positions=[{

269 - "offset": 0,

270 - "length": 3

271 - }, {

272 - "offset": 3,

273 - "length": 3

274 - }],

259 + mm_positions=[

260 + PlaceholderRange(offset=0, length=3),

261 + PlaceholderRange(offset=3, length=3),

262 + ],

263 mm_hashes=["hash3", "hash2"],

264 )

265 block_size = 3

vllm/model_executor/models/aya_vision.py

+21 -50

@@ -27,7 +27,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

27 BaseProcessingInfo,

28 MultiModalFieldConfig,

29 PromptReplacement, PromptUpdate,

30 - encode_tokens)

30 + PromptUpdateDetails)

31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

32 from vllm.sequence import IntermediateTensors

@@ -35,7 +35,6 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

35 from .siglip import SiglipVisionModel

36 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

37 maybe_prefix, merge_multimodal_embeddings)

38 -from .vision import scatter_patch_features, select_patch_features

40 class AyaVisionImagePixelInputs(TypedDict):

@@ -51,13 +50,6 @@ class AyaVisionImagePixelInputs(TypedDict):

50 num_patches: torch.Tensor

51 """Shape: `(batch_size * num_images)`"""

54 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

55 - """

56 - A boolean mask indicating which image embeddings correspond to patch tokens.

57 -

58 - Shape: `(batch_size * num_images, num_embeds)`

59 - """

60 -

54 class AyaVisionMultiModalProjector(nn.Module):

@@ -135,21 +127,20 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):

127 def get_max_image_tokens(self) -> int:

128 hf_processor = self.get_hf_processor()

129 image_processor = hf_processor.image_processor

130 +

131 image_size = self.get_image_size_with_most_features()

139 - tokenizer = hf_processor.tokenizer

132 num_patches = self.get_num_patches(

133 image_width=image_size.width,

134 image_height=image_size.height,

135 size=image_processor.size,

136 min_patches=image_processor.min_patches,

145 - max_patches=image_processor.max_patches)

146 - image_string = hf_processor._prompt_split_image(num_patches)

147 - x = encode_tokens(

148 - tokenizer,

149 - image_string,

150 - add_special_tokens=False,

137 + max_patches=image_processor.max_patches,

138 )

152 - return len(x)

139 +

140 + img_patches_per_tile = (hf_processor.img_size //

141 + hf_processor.patch_size)**2

142 +

143 + return num_patches * img_patches_per_tile

144

145 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:

146 return {"image": None}

@@ -221,7 +212,6 @@ class AyaVisionMultiModalProcessor(

212 hf_processor = self.info.get_hf_processor(**mm_kwargs)

213 image_processor = hf_processor.image_processor

214

224 - hf_config = self.info.get_hf_config()

215 # HF processor pops the `num_patches` kwarg, which is needed by vLLM

216 if (images :=

217 mm_data.get("images")) is not None and '<image>' in prompt:

@@ -234,6 +224,7 @@ class AyaVisionMultiModalProcessor(

224 parsed_images.get_image_size(i)

225 for i in range(len(parsed_images))

226 ]

227 +

228 num_patches = [

229 self.info.get_num_patches(

230 image_width=image_size.width,

@@ -243,20 +234,6 @@ class AyaVisionMultiModalProcessor(

234 max_patches=image_processor.max_patches)

235 for image_size in image_sizes

236 ]

246 - image_tokens_list = [

247 - hf_processor._prompt_split_image(num_patch)

248 - for num_patch in num_patches

249 - ]

250 - tokenizer = self.info.get_tokenizer()

251 - image_token_ids = [

252 - tokenizer.encode(image_tokens, add_special_tokens=False)

253 - for image_tokens in image_tokens_list

254 - ]

255 - embed_is_patch = [

256 - torch.tensor(image_repl_tokens) == hf_config.image_token_index

257 - for image_repl_tokens in image_token_ids

258 - ]

259 - processed_outputs["embed_is_patch"] = embed_is_patch

237 processed_outputs["num_patches"] = torch.tensor(num_patches)

238

239 return processed_outputs

@@ -271,7 +248,6 @@ class AyaVisionMultiModalProcessor(

248 pixel_values=MultiModalFieldConfig.flat_from_sizes(

249 "image", num_patches),

250 num_patches=MultiModalFieldConfig.batched("image"),

274 - embed_is_patch=MultiModalFieldConfig.batched("image"),

251 image_embeds=MultiModalFieldConfig.batched("image"),

252 )

253

@@ -283,6 +259,7 @@ class AyaVisionMultiModalProcessor(

259 ) -> Sequence[PromptUpdate]:

260 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

261 image_token = hf_processor.image_token

262 + img_patch_token = hf_processor.img_patch_token

263 image_processor = hf_processor.image_processor

264

265 def get_replacement(item_idx: int):

@@ -294,8 +271,11 @@ class AyaVisionMultiModalProcessor(

271 image_height=image_size.height,

272 size=image_processor.size,

273 min_patches=image_processor.min_patches,

297 - max_patches=image_processor.max_patches)

298 - return hf_processor._prompt_split_image(num_patches=num_patches)

274 + max_patches=image_processor.max_patches,

275 + )

276 + repl = hf_processor._prompt_split_image(num_patches=num_patches)

277 +

278 + return PromptUpdateDetails.select_text(repl, img_patch_token)

279

280 return [

281 PromptReplacement(

@@ -424,7 +404,6 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

404 self, **kwargs: object) -> Optional[AyaVisionImagePixelInputs]:

405 pixel_values = kwargs.pop("pixel_values", None)

406 num_patches = kwargs.pop("num_patches", None)

427 - embed_is_patch = kwargs.pop("embed_is_patch", None)

407 image_embeds = kwargs.pop("image_embeds", None)

408 assert image_embeds is None, "Aya Vision does not support image_embeds."

409

@@ -436,18 +415,13 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

415 raise ValueError("Incorrect type of num_patches. "

416 f"Got type: {type(num_patches)}")

417

439 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

440 - raise ValueError("Incorrect type of embed_is_patch. "

441 - f"Got type: {type(embed_is_patch)}")

442 -

418 pixel_values = flatten_bn(pixel_values, concat=True)

419 num_patches = flatten_bn(num_patches, concat=True)

445 - embed_is_patch = flatten_bn(embed_is_patch)

420 +

421 return AyaVisionImagePixelInputs(

422 type="pixel_values",

423 pixel_values=self._validate_pixel_values(pixel_values),

424 num_patches=num_patches,

450 - embed_is_patch=embed_is_patch,

425 )

426

427 def get_multimodal_embeddings(

@@ -455,11 +429,8 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

429 image_input = self._parse_and_validate_image_input(**kwargs)

430 if image_input is None:

431 return None

458 - image_features = self._process_image_input(image_input, **kwargs)

459 - return scatter_patch_features(

460 - image_features,

461 - image_input["embed_is_patch"],

462 - )

432 +

433 + return self._process_image_input(image_input, **kwargs)

434

435 def get_input_embeddings(

436 self,

@@ -471,9 +442,9 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal,

442 inputs_embeds = merge_multimodal_embeddings(

443 input_ids=input_ids,

444 inputs_embeds=inputs_embeds,

474 - multimodal_embeddings=select_patch_features(

475 - multimodal_embeddings),

476 - placeholder_token_id=self.config.image_token_index)

445 + multimodal_embeddings=multimodal_embeddings,

446 + placeholder_token_id=self.config.image_token_index,

447 + )

448

449 return inputs_embeds

450

vllm/model_executor/models/chameleon.py

+3 -3

@@ -162,9 +162,9 @@ class ChameleonMultiModalProcessor(

162 PromptReplacement(

163 modality="image",

164 target=[image_token_id],

165 - replacement=PromptUpdateDetails(

166 - full=([image_start_id] + image_tokens + [image_end_id]),

167 - features=image_tokens,

165 + replacement=PromptUpdateDetails.select_token_id(

166 + [image_start_id] + image_tokens + [image_end_id],

167 + embed_token_id=image_token_id,

168 ),

169 )

170 ]

vllm/model_executor/models/fuyu.py

+33 -52

@@ -18,7 +18,7 @@

18 """ PyTorch Fuyu model."""

19 import math

20 from collections.abc import Iterable, Mapping, Sequence

21 -from typing import Literal, Optional, Set, Tuple, TypedDict, Union

21 +from typing import Literal, Optional, Set, Tuple, TypedDict

23 import torch

24 import torch.nn as nn

@@ -43,7 +43,6 @@ from vllm.sequence import IntermediateTensors

43 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

44 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

45 merge_multimodal_embeddings)

46 -from .vision import scatter_patch_features, select_patch_features

47 # Cannot find the following 2 numbers from hf config.

48 _IMAGE_TOKEN_ID = 71011

@@ -66,14 +65,6 @@ class FuyuImagePatchInputs(TypedDict):

65 flattened just like `flat_data`.

66 """

69 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

70 - """

71 - A boolean mask indicating which image embeddings correspond

72 - to patch tokens.

73 -

74 - Shape: `(batch_size * num_images, num_embeds)`

75 - """

76 -

69 class FuyuProcessingInfo(BaseProcessingInfo):

@@ -94,15 +85,7 @@ class FuyuProcessingInfo(BaseProcessingInfo):

85 seq_len: int,

86 mm_counts: Mapping[str, int],

87 ) -> Mapping[str, int]:

97 - target_width, target_height = self.get_image_size_with_most_features()

98 -

99 - max_ncols, max_nrows = self.get_image_feature_grid_size(

100 - image_width=target_width,

101 - image_height=target_height,

102 - )

103 - max_image_tokens = (max_ncols + 1) * max_nrows

104 -

105 - return {"image": max_image_tokens}

88 + return {"image": self.get_max_image_tokens()}

90 def get_image_feature_grid_size(

91 self,

@@ -128,11 +111,32 @@ class FuyuProcessingInfo(BaseProcessingInfo):

111 nrows = math.ceil(image_height / patch_height)

112 return ncols, nrows

113

114 + def get_num_image_tokens(

115 + self,

116 + *,

117 + image_width: int,

118 + image_height: int,

119 + ) -> int:

120 + ncols, nrows = self.get_image_feature_grid_size(

121 + image_width=image_width,

122 + image_height=image_height,

123 + )

124 +

125 + return ncols * nrows

126 +

127 def get_image_size_with_most_features(self) -> ImageSize:

128 image_processor = self.get_image_processor()

129 return ImageSize(width=image_processor.size["width"],

130 height=image_processor.size["height"])

131

132 + def get_max_image_tokens(self) -> int:

133 + target_width, target_height = self.get_image_size_with_most_features()

134 +

135 + return self.get_num_image_tokens(

136 + image_width=target_width,

137 + image_height=target_height,

138 + )

139 +

140

141 class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):

142

@@ -192,19 +196,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

196

197 processed_outputs["image_patches"] = image_patches[0]

198

195 - # get patch grid size for each image

196 - embed_is_patch = []

197 - for image in images:

198 - ncols, nrows = self.info.get_image_feature_grid_size(

199 - image_width=image.width,

200 - image_height=image.height,

201 - )

202 -

203 - mask = torch.tensor(([True] * ncols + [False]) * nrows)

204 - embed_is_patch.append(mask)

205 -

206 - processed_outputs["embed_is_patch"] = embed_is_patch

207 -

199 return processed_outputs

200

201 def _apply_hf_processor_tokens_only(

@@ -224,8 +215,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

215 hf_inputs: BatchFeature,

216 hf_processor_mm_kwargs: Mapping[str, object],

217 ) -> Mapping[str, MultiModalFieldConfig]:

227 - return dict(image_patches=MultiModalFieldConfig.batched("image"),

228 - embed_is_patch=MultiModalFieldConfig.batched("image"))

218 + return dict(image_patches=MultiModalFieldConfig.batched("image"))

219

220 def _get_prompt_updates(

221 self,

@@ -252,9 +242,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):

242 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +

243 [_NEWLINE_TOKEN_ID]) * nrows

244

255 - return PromptUpdateDetails(

256 - full=image_tokens + [bos_token_id],

257 - features=image_tokens,

245 + return PromptUpdateDetails.select_token_id(

246 + image_tokens + [bos_token_id],

247 + embed_token_id=_IMAGE_TOKEN_ID,

248 )

249

250 return [

@@ -329,20 +319,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

319 raise ValueError("Incorrect type of image patches. "

320 f"Got type: {type(image_patches)}")

321

332 - embed_is_patch = kwargs.pop("embed_is_patch")

333 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

334 - raise ValueError("Incorrect type of embed_is_patch. "

335 - f"Got type: {type(embed_is_patch)}")

336 -

322 image_patches_flat = flatten_bn(image_patches)

338 - embed_is_patch = flatten_bn(embed_is_patch)

323

324 return FuyuImagePatchInputs(

325 type="image_patches",

326 flat_data=self._validate_pixel_values(

327 flatten_bn(image_patches_flat, concat=True)),

328 patches_per_image=[x.size(0) for x in image_patches_flat],

345 - embed_is_patch=embed_is_patch,

329 )

330

331 return None

@@ -364,12 +347,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

347 if image_input is None:

348 return None

349

367 - image_features = self._process_image_input(image_input)

368 -

369 - return scatter_patch_features(

370 - image_features,

371 - image_input["embed_is_patch"],

372 - )

350 + return self._process_image_input(image_input)

351

352 def get_input_embeddings(

353 self,

@@ -379,8 +357,11 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

357 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

358 if multimodal_embeddings is not None:

359 inputs_embeds = merge_multimodal_embeddings(

382 - input_ids, inputs_embeds,

383 - select_patch_features(multimodal_embeddings), _IMAGE_TOKEN_ID)

360 + input_ids,

361 + inputs_embeds,

362 + multimodal_embeddings,

363 + _IMAGE_TOKEN_ID,

364 + )

365 return inputs_embeds

366

367 def forward(

vllm/model_executor/models/gemma3_mm.py

+20 -63

@@ -25,7 +25,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

25 PlaceholderFeaturesInfo,

26 PromptReplacement, PromptTargetMatch,

27 PromptUpdate, PromptUpdateDetails,

28 - encode_tokens, find_mm_placeholders,

28 + find_mm_placeholders,

29 replace_token_matches)

30 # yapf: enable

31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

@@ -36,7 +36,6 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

36 from .siglip import SiglipVisionModel

37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

38 maybe_prefix, merge_multimodal_embeddings)

39 -from .vision import scatter_patch_features, select_patch_features

40 logger = init_logger(__name__)

@@ -54,14 +53,6 @@ class Gemma3ImagePixelInputs(TypedDict):

53 num_patches: torch.Tensor

54 """Shape: `(batch_size * num_images)`"""

57 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

58 - """

59 - A boolean mask indicating which image embeddings correspond

60 - to patch tokens.

61 -

62 - Shape: `(batch_size * num_images, num_embeds)`

63 - """

64 -

57 Gemma3ImageInputs = Gemma3ImagePixelInputs

@@ -183,7 +174,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

174 if processor is None:

175 processor = self.get_hf_processor()

176

186 - image_token = processor.boi_token

177 + boi_token = processor.boi_token

178

179 num_crops = self.get_num_crops(

180 image_width=image_width,

@@ -192,19 +183,21 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

183 )

184

185 if num_crops == 0:

195 - image_text = image_token

186 + image_text = boi_token

187 else:

197 - crops_image_tokens = " ".join(image_token

198 - for _ in range(num_crops))

188 + crops_image_tokens = " ".join(boi_token for _ in range(num_crops))

189 image_text = (

200 - f"Here is the original image {image_token} and here are some "

190 + f"Here is the original image {boi_token} and here are some "

191 f"crops to help you see better {crops_image_tokens}")

192

203 - repl_full = image_text.replace(image_token,

193 + repl_full = image_text.replace(boi_token,

194 processor.full_image_sequence)

205 - repl_features = repl_full.strip("\n")

195

207 - return PromptUpdateDetails(full=repl_full, features=repl_features)

196 + tokenizer = processor.tokenizer

197 + vocab = tokenizer.get_vocab()

198 + image_token_id = vocab[tokenizer.image_token]

199 +

200 + return PromptUpdateDetails.select_token_id(repl_full, image_token_id)

201

202 def get_num_image_tokens(

203 self,

@@ -213,19 +206,17 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

206 image_height: int,

207 processor: Optional[Gemma3Processor],

208 ) -> int:

216 - tokenizer = self.get_tokenizer()

217 - image_repl = self.get_image_repl(

209 + if processor is None:

210 + processor = self.get_hf_processor()

211 +

212 + num_crops = self.get_num_crops(

213 image_width=image_width,

214 image_height=image_height,

215 processor=processor,

216 )

217 + image_seq_len = processor.image_seq_length

218

223 - image_repl_tokens = encode_tokens(

224 - tokenizer,

225 - image_repl.features,

226 - add_special_tokens=False,

227 - )

228 - return len(image_repl_tokens)

219 + return (num_crops + 1) * image_seq_len

220

221 def get_image_size_with_most_features(self) -> ImageSize:

222 processor = self.get_hf_processor()

@@ -301,28 +292,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

292 ]

293 hf_processor = self.info.get_hf_processor(**mm_kwargs)

294

304 - image_repl_features = [

305 - self.info.get_image_repl(image_width=size.width,

306 - image_height=size.height,

307 - processor=hf_processor).features

308 - for size in image_sizes

309 - ]

310 -

311 - tokenizer = self.info.get_tokenizer()

312 - image_repls_feature_tokens = [

313 - tokenizer.encode(image_repl, add_special_tokens=False)

314 - for image_repl in image_repl_features

315 - ]

316 -

317 - vocab = tokenizer.get_vocab()

318 - image_token_id = vocab[tokenizer.image_token]

319 -

320 - embed_is_patch = [

321 - torch.tensor(image_repl_tokens) == image_token_id

322 - for image_repl_tokens in image_repls_feature_tokens

323 - ]

324 - processed_outputs["embed_is_patch"] = embed_is_patch

325 -

295 num_crops = [

296 self.info.get_num_crops(image_width=size.width,

297 image_height=size.height,

@@ -344,7 +313,6 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

313 pixel_values=MultiModalFieldConfig.flat_from_sizes(

314 "image", num_crops + 1),

315 num_crops=MultiModalFieldConfig.batched("image"),

347 - embed_is_patch=MultiModalFieldConfig.batched("image"),

316 )

317

318 def _get_prompt_updates(

@@ -454,6 +422,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

422 item_idx=p.item_idx,

423 start_idx=repl_orig_idxs[p.start_idx],

424 tokens=p.tokens,

425 + is_embed=p.is_embed,

426 ) for p in placeholders

427 ]

428 for modality, placeholders in repls.items()

@@ -572,7 +541,6 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

541 self, **kwargs: object) -> Optional[Gemma3ImageInputs]:

542 pixel_values = kwargs.pop("pixel_values", None)

543 num_crops = kwargs.pop("num_crops", None)

575 - embed_is_patch = kwargs.pop("embed_is_patch", None)

544 image_embeds = kwargs.pop("image_embeds", None)

545 assert image_embeds is None, "Gemma3 does not support image_embeds."

546 if pixel_values is None:

@@ -586,19 +554,13 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

554 raise ValueError("Incorrect type of num_crops. "

555 f"Got type: {type(num_crops)}")

556

589 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

590 - raise ValueError("Incorrect type of embed_is_patch. "

591 - f"Got type: {type(embed_is_patch)}")

592 -

557 pixel_values = flatten_bn(pixel_values, concat=True)

558 num_crops = flatten_bn(num_crops, concat=True)

595 - embed_is_patch = flatten_bn(embed_is_patch)

559

560 return Gemma3ImagePixelInputs(

561 type="pixel_values",

562 pixel_values=self._validate_pixel_values(pixel_values),

563 num_patches=num_crops + 1,

601 - embed_is_patch=embed_is_patch,

564 )

565

566 def _image_pixels_to_features(

@@ -635,12 +597,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

597 if image_input is None:

598 return None

599

638 - image_features = self._process_image_input(image_input)

639 -

640 - return scatter_patch_features(

641 - image_features,

642 - image_input["embed_is_patch"],

643 - )

600 + return self._process_image_input(image_input)

601

602 def get_input_embeddings(

603 self,

@@ -652,7 +609,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

609 inputs_embeds = merge_multimodal_embeddings(

610 input_ids,

611 inputs_embeds,

655 - select_patch_features(multimodal_embeddings),

612 + multimodal_embeddings,

613 self.config.image_token_index,

614 )

615 return inputs_embeds

vllm/model_executor/models/h2ovl.py

+1 -1

@@ -257,7 +257,7 @@ class H2OVLProcessor(BaseInternVLProcessor):

257 repl_features = IMG_CONTEXT * feature_size

258 repl_full = IMG_START + repl_features + IMG_END

259

260 - return PromptUpdateDetails(full=repl_full, features=repl_features)

260 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)

261

262 def resolve_min_max_num(

263 self,

vllm/model_executor/models/idefics3.py

+15 -67

@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,

41 MultiModalDataItems,

42 MultiModalFieldConfig,

43 PromptReplacement, PromptUpdate,

44 - encode_tokens)

44 + PromptUpdateDetails)

45 # yapf: enable

46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

47 from vllm.sequence import IntermediateTensors

@@ -54,7 +54,6 @@ from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal

54 from .llama import LlamaModel

55 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

56 merge_multimodal_embeddings)

57 -from .vision import scatter_patch_features, select_patch_features

59 class Idefics3ImagePixelInputs(TypedDict):

@@ -69,14 +68,6 @@ class Idefics3ImagePixelInputs(TypedDict):

68 num_patches: torch.Tensor

69 """Shape: `(batch_size * num_images)`"""

72 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

73 - """

74 - A boolean mask indicating which image embeddings correspond

75 - to patch tokens.

76 -

77 - Shape: `(batch_size * num_images, num_embeds)`

78 - """

79 -

72 class Idefics3ImageEmbeddingInputs(TypedDict):

73 type: Literal["image_embeds"]

@@ -86,14 +77,6 @@ class Idefics3ImageEmbeddingInputs(TypedDict):

77 `hidden_size` must match the hidden size of language model backbone.

78 """

89 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

90 - """

91 - A boolean mask indicating which image embeddings correspond

92 - to patch tokens.

93 -

94 - Shape: `(batch_size * num_images, num_embeds)`

95 - """

96 -

81 ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]

@@ -275,19 +258,16 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):

258 image_height: int,

259 processor: Optional[Idefics3Processor],

260 ) -> int:

278 - tokenizer = self.get_tokenizer()

279 - image_repl = self.get_image_repl(

261 + if processor is None:

262 + processor = self.get_hf_processor()

263 +

264 + num_patches = self.get_num_patches(

265 image_width=image_width,

266 image_height=image_height,

267 processor=processor,

268 )

269

285 - image_repl_tokens = encode_tokens(

286 - tokenizer,

287 - image_repl,

288 - add_special_tokens=False,

289 - )

290 - return len(image_repl_tokens)

270 + return num_patches * processor.image_seq_len

271

272 def get_image_size_with_most_features(self) -> ImageSize:

273 processor = self.get_hf_processor()

@@ -364,28 +344,6 @@ class Idefics3MultiModalProcessor(

344 ]

345 hf_processor = self.info.get_hf_processor(**mm_kwargs)

346

367 - image_repl_features = [

368 - self.info.get_image_repl(image_width=size.width,

369 - image_height=size.height,

370 - processor=hf_processor)

371 - for size in image_sizes

372 - ]

373 -

374 - tokenizer = self.info.get_tokenizer()

375 - image_repls_feature_tokens = [

376 - tokenizer.encode(image_repl, add_special_tokens=False)

377 - for image_repl in image_repl_features

378 - ]

379 -

380 - vocab = tokenizer.get_vocab()

381 - image_token_id = vocab[hf_processor.image_token.content]

382 -

383 - embed_is_patch = [

384 - torch.tensor(image_repl_tokens) == image_token_id

385 - for image_repl_tokens in image_repls_feature_tokens

386 - ]

387 - processed_outputs["embed_is_patch"] = embed_is_patch

388 -

347 num_patches = [

348 self.info.get_num_patches(

349 image_width=size.width,

@@ -415,7 +373,6 @@ class Idefics3MultiModalProcessor(

373 "image", num_patches),

374 image_embeds=MultiModalFieldConfig.batched("image"),

375 num_patches=MultiModalFieldConfig.batched("image"),

418 - embed_is_patch=MultiModalFieldConfig.batched("image"),

376 )

377

378 def _get_prompt_updates(

@@ -427,17 +384,22 @@ class Idefics3MultiModalProcessor(

384 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

385 image_token = hf_processor.image_token.content

386

430 - def get_replacement_idefics3(item_idx: int) -> str:

387 + def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:

388 images = mm_items.get_items("image", ImageProcessorItems)

389

390 image_size = images.get_image_size(item_idx)

391

435 - return self.info.get_image_repl(

392 + image_repl = self.info.get_image_repl(

393 image_width=image_size.width,

394 image_height=image_size.height,

395 processor=hf_processor,

396 )

397

398 + return PromptUpdateDetails.select_text(

399 + image_repl,

400 + embed_text=image_token,

401 + )

402 +

403 return [

404 PromptReplacement(

405 modality="image",

@@ -675,13 +637,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

637 if pixel_values is None and image_embeds is None:

638 return None

639

678 - embed_is_patch = kwargs.pop("embed_is_patch")

679 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

680 - raise ValueError("Incorrect type of embed_is_patch. "

681 - f"Got type: {type(embed_is_patch)}")

682 -

683 - embed_is_patch = flatten_bn(embed_is_patch)

684 -

640 if image_embeds is not None:

641 if not isinstance(image_embeds, (torch.Tensor, list)):

642 raise ValueError("Incorrect type of image embeddings. "

@@ -690,7 +645,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

645 return Idefics3ImageEmbeddingInputs(

646 type="image_embeds",

647 data=flatten_bn(image_embeds, concat=True),

693 - embed_is_patch=embed_is_patch,

648 )

649

650 if pixel_values is not None:

@@ -718,7 +672,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

672 pixel_values=self._validate_pixel_values(pixel_values),

673 pixel_attention_mask=pixel_attention_mask,

674 num_patches=num_patches,

721 - embed_is_patch=embed_is_patch,

675 )

676

677 raise AssertionError("This line should be unreachable.")

@@ -754,12 +707,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

707 if image_input is None:

708 return None

709

757 - image_features = self._process_image_input(image_input)

758 -

759 - return scatter_patch_features(

760 - image_features,

761 - image_input["embed_is_patch"],

762 - )

710 + return self._process_image_input(image_input)

711

712 def get_input_embeddings(

713 self,

@@ -771,7 +719,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,

719 inputs_embeds = merge_multimodal_embeddings(

720 input_ids,

721 inputs_embeds,

774 - select_patch_features(multimodal_embeddings),

722 + multimodal_embeddings,

723 self.config.image_token_id,

724 )

725 return inputs_embeds

vllm/model_executor/models/internvl.py

+3 -40

@@ -39,7 +39,6 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer

39 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

40 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

41 maybe_prefix, merge_multimodal_embeddings)

42 -from .vision import scatter_patch_features, select_patch_features

43 IMG_START = '<img>'

44 IMG_END = '</img>'

@@ -60,14 +59,6 @@ class InternVLImagePixelInputs(TypedDict):

59 num_patches: torch.Tensor

60 """Shape: `(batch_size * num_images)`"""

63 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

64 - """

65 - A boolean mask indicating which image embeddings correspond

66 - to patch tokens.

67 -

68 - Shape: `(batch_size * num_images, num_embeds)`

69 - """

70 -

63 class InternVLImageEmbeddingInputs(TypedDict):

64 type: Literal["image_embeds"]

@@ -419,24 +410,12 @@ class BaseInternVLProcessor(ABC):

410 torch.tensor([len(item) for item in pixel_values_lst]),

411 }

412

422 - tokenizer = self.tokenizer

423 - image_token_id = self.image_token_id

424 -

425 - embed_is_patch = list[torch.Tensor]()

426 -

413 for pixel_values in pixel_values_lst:

414 num_patches = pixel_values.shape[0]

415 feature_size = num_patches * self.num_image_token

416

417 image_repl = self.get_image_repl(feature_size, num_patches)

432 - feature_tokens = tokenizer.encode(image_repl.features,

433 - add_special_tokens=False)

434 -

418 text = [t.replace('<image>', image_repl.full, 1) for t in text]

436 - embed_is_patch.append(

437 - torch.tensor(feature_tokens) == image_token_id)

438 -

439 - image_inputs["embed_is_patch"] = embed_is_patch

419

420 text_inputs = self.tokenizer(text)

421

@@ -460,7 +439,7 @@ class InternVLProcessor(BaseInternVLProcessor):

439 repl_features = IMG_CONTEXT * feature_size

440 repl_full = IMG_START + repl_features + IMG_END

441

463 - return PromptUpdateDetails(full=repl_full, features=repl_features)

442 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)

443

444

445 class BaseInternVLProcessingInfo(BaseProcessingInfo):

@@ -599,7 +578,6 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

578 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(

579 "image", image_num_patches),

580 image_num_patches=MultiModalFieldConfig.batched("image"),

602 - embed_is_patch=MultiModalFieldConfig.batched("image"),

581 image_embeds=MultiModalFieldConfig.batched("image"),

582 image_token_id=MultiModalFieldConfig.shared("image", num_images),

583 )

@@ -831,7 +809,6 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

809 self, **kwargs: object) -> Optional[InternVLImageInputs]:

810 pixel_values_flat = kwargs.pop("pixel_values_flat", None)

811 image_num_patches = kwargs.pop("image_num_patches", None)

834 - embed_is_patch = kwargs.pop("embed_is_patch", None)

812 image_embeds = kwargs.pop("image_embeds", None)

813

814 if pixel_values_flat is None and image_embeds is None:

@@ -860,20 +837,14 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

837 raise ValueError("Incorrect type of image_num_patches. "

838 f"Got type: {type(image_num_patches)}")

839

863 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

864 - raise ValueError("Incorrect type of embed_is_patch. "

865 - f"Got type: {type(embed_is_patch)}")

866 -

840 pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)

841 image_num_patches = flatten_bn(image_num_patches, concat=True)

869 - embed_is_patch = flatten_bn(embed_is_patch)

842

843 return InternVLImagePixelInputs(

844 type="pixel_values",

845 pixel_values_flat=self._validate_pixel_values(

846 pixel_values_flat),

847 num_patches=image_num_patches,

876 - embed_is_patch=embed_is_patch,

848 )

849

850 raise AssertionError("This line should be unreachable.")

@@ -919,15 +890,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

890 if image_input is None:

891 return None

892

922 - image_features = self._process_image_input(image_input)

923 -

924 - if image_input["type"] != "pixel_values":

925 - return image_features

926 -

927 - return scatter_patch_features(

928 - image_features,

929 - image_input["embed_is_patch"],

930 - )

893 + return self._process_image_input(image_input)

894

895 def get_input_embeddings(

896 self,

@@ -941,7 +904,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

904 inputs_embeds = merge_multimodal_embeddings(

905 input_ids,

906 inputs_embeds,

944 - select_patch_features(multimodal_embeddings),

907 + multimodal_embeddings,

908 self.img_context_token_id,

909 )

910 return inputs_embeds

vllm/model_executor/models/llava.py

+6 -49

@@ -32,7 +32,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

32 ImageSize, MultiModalDataItems)

33 from vllm.multimodal.processing import (BaseMultiModalProcessor,

34 BaseProcessingInfo, ProcessingCache,

35 - PromptReplacement, PromptUpdate)

35 + PromptReplacement, PromptUpdate,

36 + PromptUpdateDetails)

37 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

38 from vllm.sequence import IntermediateTensors

@@ -42,8 +43,7 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel

43 from .siglip import SiglipVisionModel

44 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

45 maybe_prefix, merge_multimodal_embeddings)

45 -from .vision import (get_vision_encoder_info, scatter_patch_features,

46 - select_patch_features)

46 +from .vision import get_vision_encoder_info

49 class LlavaImagePixelInputs(TypedDict):

@@ -67,14 +67,6 @@ class PixtralHFImagePixelInputs(TypedDict):

67 in which case the data is passed as a list instead of a batched tensor.

68 """

70 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

71 - """

72 - A boolean mask indicating which image embeddings correspond

73 - to patch tokens.

74 -

75 - Shape: `(batch_size * num_images, num_embeds)`

76 - """

77 -

71 class LlavaImageEmbeddingInputs(TypedDict):

72 type: Literal["image_embeds"]

@@ -343,23 +335,6 @@ class PixtralHFMultiModalProcessor(

335 for p, (h, w) in zip(pixel_values, image_sizes)

336 ]

337

346 - hf_config = self.info.get_hf_config()

347 - vision_config = hf_config.vision_config

348 - assert isinstance(vision_config, PixtralVisionConfig)

349 - encoder_info = PixtralHFEncoderInfo(vision_config)

350 -

351 - tile_sizes = [

352 - encoder_info.get_patch_grid_size(

353 - image_width=pixel_value.shape[-1],

354 - image_height=pixel_value.shape[-2],

355 - ) for pixel_value in processed_outputs["pixel_values"]

356 - ]

357 - embed_is_patch = [

358 - torch.tensor(([True] * ncols + [False]) * nrows)

359 - for ncols, nrows in tile_sizes

360 - ]

361 - processed_outputs["embed_is_patch"] = embed_is_patch

362 -

338 return processed_outputs

339

340 def _get_mm_fields_config(

@@ -369,7 +344,6 @@ class PixtralHFMultiModalProcessor(

344 ) -> Mapping[str, MultiModalFieldConfig]:

345 return dict(

346 pixel_values=MultiModalFieldConfig.batched("image"),

372 - embed_is_patch=MultiModalFieldConfig.batched("image"),

347 image_embeds=MultiModalFieldConfig.batched("image"),

348 )

349

@@ -404,7 +378,7 @@ class PixtralHFMultiModalProcessor(

378 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows

379 tokens[-1] = image_end_id

380

407 - return tokens

381 + return PromptUpdateDetails.select_token_id(tokens, image_token_id)

382

383 return [

384 PromptReplacement(

@@ -612,17 +586,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

586 f"Got type: {type(pixel_values)}")

587

588 if self.config.vision_config.model_type == "pixtral":

615 - embed_is_patch = kwargs.pop("embed_is_patch")

616 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

617 - raise ValueError("Incorrect type of embed_is_patch. "

618 - f"Got type: {type(embed_is_patch)}")

619 -

620 - embed_is_patch = flatten_bn(embed_is_patch)

621 -

589 return PixtralHFImagePixelInputs(

590 type="pixel_values_pixtral",

591 pixel_values=flatten_bn(pixel_values),

625 - embed_is_patch=embed_is_patch,

592 )

593

594 return LlavaImagePixelInputs(

@@ -714,16 +680,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

680 if image_input is None:

681 return None

682

717 - image_features = self._process_image_input(image_input)

718 -

719 - if image_input["type"] != "pixel_values_pixtral":

720 - # The path is used for pixtral (V0 only) and llava (V0/V1)

721 - return image_features

722 -

723 - return scatter_patch_features(

724 - image_features,

725 - image_input["embed_is_patch"],

726 - )

683 + return self._process_image_input(image_input)

684

685 def get_input_embeddings(

686 self,

@@ -735,7 +692,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

692 inputs_embeds = merge_multimodal_embeddings(

693 input_ids,

694 inputs_embeds,

738 - select_patch_features(multimodal_embeddings),

695 + multimodal_embeddings,

696 self.config.image_token_index,

697 )

698 return inputs_embeds

vllm/model_executor/models/minicpmo.py

+9 -65

@@ -40,7 +40,8 @@ from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,

40 DictEmbeddingItems, ModalityData,

41 ModalityDataItems, MultiModalDataItems,

42 MultiModalDataParser)

43 -from vllm.multimodal.processing import PromptReplacement, PromptUpdate

43 +from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,

44 + PromptUpdateDetails)

45 from vllm.multimodal.profiling import ProcessorInputs

47 from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,

@@ -50,7 +51,6 @@ from .minicpmv import (_MAX_FRAMES_PER_VIDEO, MiniCPMV2_6,

51 _minicpmv_field_config)

52 from .utils import (AutoWeightsLoader, cast_overflow_tensors, flatten_bn,

53 maybe_prefix)

53 -from .vision import scatter_patch_features

55 CPU_DEVICE = torch.device("cpu")

@@ -73,14 +73,6 @@ class MiniCPMOAudioFeatureInputs(TypedDict):

73 which equals to `audio_features.shape[-1]`

74 """

76 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

77 - """

78 - A boolean mask indicating which audio embeddings correspond

79 - to patch tokens.

80 -

81 - Shape: `(batch_size * num_audios, num_embeds)`

82 - """

83 -

77 class MiniCPMOAudioEmbeddingInputs(TypedDict):

78 type: Literal["audio_embeds"]

@@ -93,14 +85,6 @@ class MiniCPMOAudioEmbeddingInputs(TypedDict):

85 Length of each slice may vary, so pass it as a list.

86 """

96 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

97 - """

98 - A boolean mask indicating which audio embeddings correspond

99 - to patch tokens.

100 -

101 - Shape: `(batch_size * num_audios, num_embeds)`

102 - """

103 -

89 MiniCPMOAudioInputs = Union[MiniCPMOAudioFeatureInputs,

90 MiniCPMOAudioEmbeddingInputs]

@@ -115,7 +99,6 @@ def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):

99 audio_features=MultiModalFieldConfig.batched("audio"),

100 audio_feature_lens=MultiModalFieldConfig.batched("audio"),

101 audio_embeds=MultiModalFieldConfig.batched("audio"),

118 - audio_embed_is_patch=MultiModalFieldConfig.batched("audio"),

102 audio_token_id=MultiModalFieldConfig.shared("audio", num_audios),

103 )

104

@@ -197,8 +180,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):

180 pool_step = self.get_default_audio_pool_step()

181 fbank_feat_in_chunk = 100

182 cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1

200 - num_audio_tokens = (cnn_feat_in_chunk - pool_step) // pool_step + 1

201 - return num_audio_tokens + 2 # <audio>(<unk>*N)</audio>

183 + return (cnn_feat_in_chunk - pool_step) // pool_step + 1

184

185 def get_max_audio_chunks_with_most_features(self) -> int:

186 return 30

@@ -209,8 +191,7 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):

191

192 def get_audio_len_by_num_chunks(self, num_chunks: int) -> int:

193 sampling_rate = self.get_default_audio_sampling_rate()

212 - # exclude <audio> </audio>

213 - num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk() - 2

194 + num_tokens_per_chunk = self.get_max_audio_tokens_per_chunk()

195 return int(num_chunks * sampling_rate / num_tokens_per_chunk) + 1

196

197 def get_num_frames_with_most_features(

@@ -295,13 +276,6 @@ class MiniCPMOMultiModalProcessor(

276

277 if isinstance(parsed_audios, MiniCPMOAudioEmbeddingItems):

278 audio_inputs = {}

298 -

299 - audio_lens = [

300 - self.info.get_audio_len_by_num_chunks(

301 - sum(map(len,

302 - parsed_audios.get(i)["audio_embeds"])))

303 - for i in range(len(parsed_audios))

304 - ]

279 else:

280 audio_inputs = self._base_call_hf_processor(

281 prompts=[self.info.audio_pattern] * len(parsed_audios),

@@ -323,27 +297,7 @@ class MiniCPMOMultiModalProcessor(

297 ]

298 audio_inputs["audio_features"] = unpadded_audio_features

299

326 - audio_lens = [

327 - parsed_audios.get_audio_length(i)

328 - for i in range(len(parsed_audios))

329 - ]

330 -

331 - audio_repl_features = [

332 - self.get_audio_prompt_texts(audio_len) for audio_len in audio_lens

333 - ]

334 -

300 tokenizer = self.info.get_tokenizer()

336 - audio_repls_feature_tokens = [

337 - tokenizer.encode(audio_repl, add_special_tokens=False)

338 - for audio_repl in audio_repl_features

339 - ]

340 -

341 - embed_is_patch = [

342 - self.get_embed_is_patch(audio_repl_tokens)

343 - for audio_repl_tokens in audio_repls_feature_tokens

344 - ]

345 - audio_inputs["audio_embed_is_patch"] = embed_is_patch

346 -

301 unk_token_id = tokenizer.get_vocab()["<unk>"]

302 audio_inputs["audio_token_id"] = torch.tensor(unk_token_id)

303

@@ -384,7 +338,10 @@ class MiniCPMOMultiModalProcessor(

338 else:

339 audio_len = audios.get_audio_length(item_idx)

340

387 - return self.get_audio_prompt_texts(audio_len)

341 + return PromptUpdateDetails.select_text(

342 + self.get_audio_prompt_texts(audio_len),

343 + "<unk>",

344 + )

345

346 return [

347 *base_updates,

@@ -713,13 +670,6 @@ class MiniCPMO(MiniCPMV2_6):

670 assert isinstance(audio_token_id, torch.Tensor)

671 self.mm_token_ids.add(audio_token_id.flatten().unique().item())

672

716 - audio_embed_is_patch = kwargs.pop("audio_embed_is_patch")

717 - if not isinstance(audio_embed_is_patch, (torch.Tensor, list)):

718 - raise ValueError("Incorrect type of audio_embed_is_patch. "

719 - f"Got type: {type(audio_embed_is_patch)}")

720 -

721 - audio_embed_is_patch = flatten_bn(audio_embed_is_patch)

722 -

673 if audio_embeds is not None:

674 if not isinstance(audio_embeds, (torch.Tensor, list)):

675 raise ValueError("Incorrect type of audio_embeds. "

@@ -730,7 +680,6 @@ class MiniCPMO(MiniCPMV2_6):

680 return MiniCPMOAudioEmbeddingInputs(

681 type="audio_embeds",

682 audio_embeds=audio_embeds_flat,

733 - embed_is_patch=audio_embed_is_patch,

683 )

684

685 if not isinstance(audio_features, (torch.Tensor, list)):

@@ -749,7 +698,6 @@ class MiniCPMO(MiniCPMV2_6):

698 type="audio_features",

699 audio_features=audio_features_flat,

700 audio_feature_lens=audio_feature_lens_flat,

752 - embed_is_patch=audio_embed_is_patch,

701 )

702

703 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

@@ -781,10 +729,6 @@ class MiniCPMO(MiniCPMV2_6):

729 if modality == "audios":

730 audio_input = modalities["audios"]

731 audio_features = self._process_audio_input(audio_input)

784 - multimodal_embeddings += tuple(

785 - scatter_patch_features(

786 - audio_features,

787 - audio_input["embed_is_patch"],

788 - ))

732 + multimodal_embeddings += tuple(audio_features)

733

734 return multimodal_embeddings

vllm/model_executor/models/minicpmv.py

+41 -101

@@ -56,7 +56,7 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,

56 VideoItem, VideoProcessorItems)

57 from vllm.multimodal.processing import (BaseMultiModalProcessor,

58 BaseProcessingInfo, PromptReplacement,

59 - PromptUpdate)

59 + PromptUpdate, PromptUpdateDetails)

60 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

61 from vllm.platforms import current_platform

62 from vllm.sequence import IntermediateTensors

@@ -67,7 +67,6 @@ from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

67 SupportsMultiModal, SupportsPP)

68 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

69 merge_multimodal_embeddings)

70 -from .vision import scatter_patch_features, select_patch_features

71 # For profile run

72 _MAX_FRAMES_PER_VIDEO = 16

@@ -90,14 +89,6 @@ class MiniCPMVImagePixelInputs(TypedDict):

89 This should be in `(height, width)` format.

90 """

93 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

94 - """

95 - A boolean mask indicating which image embeddings correspond

96 - to patch tokens.

97 -

98 - Shape: `(batch_size * num_images, num_embeds)`

99 - """

100 -

92 num_slices: torch.Tensor

93 """Shape: `(batch_size * num_images)`"""

@@ -112,14 +103,6 @@ class MiniCPMVImageEmbeddingInputs(TypedDict):

103 instead of a batched tensor.

104 """

105

115 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

116 - """

117 - A boolean mask indicating which image embeddings correspond

118 - to patch tokens.

119 -

120 - Shape: `(batch_size * num_images, num_embeds)`

121 - """

122 -

106

107 MiniCPMVImageInputs = Union[MiniCPMVImagePixelInputs,

108 MiniCPMVImageEmbeddingInputs]

@@ -245,12 +228,10 @@ def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):

228 image_sizes=MultiModalFieldConfig.batched("image"),

229 tgt_sizes=MultiModalFieldConfig.batched("image"),

230 image_embeds=MultiModalFieldConfig.batched("image"),

248 - embed_is_patch=MultiModalFieldConfig.batched("image"),

231 video_pixel_values=MultiModalFieldConfig.batched("video"),

232 video_image_sizes=MultiModalFieldConfig.batched("video"),

233 video_tgt_sizes=MultiModalFieldConfig.batched("video"),

234 video_embeds=MultiModalFieldConfig.batched("video"),

253 - video_embed_is_patch=MultiModalFieldConfig.batched("video"),

235 image_token_id=MultiModalFieldConfig.shared("image", num_images),

236 video_token_id=MultiModalFieldConfig.shared("video", num_videos),

237 )

@@ -398,22 +379,43 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

379 use_image_id=use_image_id,

380 )

381

382 + def get_sliced_grid(

383 + self,

384 + image_size: ImageSize,

385 + # For MiniCPM V/O 2.6

386 + max_slice_nums: Optional[int] = None,

387 + ) -> Optional[tuple[int, int]]:

388 + image_processor = self.get_image_processor()

389 + version = self.get_model_version()

390 +

391 + if version == (2, 0) or version == (2, 5):

392 + return image_processor.get_sliced_grid(image_size)

393 +

394 + if max_slice_nums is None:

395 + max_slice_nums = image_processor.max_slice_nums

396 +

397 + return image_processor.get_sliced_grid(

398 + image_size,

399 + max_slice_nums=max_slice_nums,

400 + )

401 +

402 def get_num_image_tokens(

403 self,

404 image_size: ImageSize,

405 max_slice_nums: Optional[int] = None,

405 - use_image_id: bool = True,

406 ) -> int:

407 - tokenizer = self.get_tokenizer()

408 - image_placeholders = self.get_slice_image_placeholder(

407 + image_processor = self.get_image_processor()

408 +

409 + grid = self.get_sliced_grid(

410 image_size,

411 max_slice_nums=max_slice_nums,

411 - use_image_id=use_image_id,

412 )

413 - image_token_ids = tokenizer.encode(image_placeholders,

414 - add_special_tokens=False)

413 + if grid is None:

414 + ncols = nrows = 0

415 + else:

416 + ncols, nrows = grid

417

416 - return len(image_token_ids)

418 + return (ncols * nrows + 1) * image_processor.image_feature_size

419

420 def get_max_image_tokens(self) -> int:

421 image_size = self.get_image_size_with_most_features()

@@ -433,7 +435,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

435 return self.get_num_image_tokens(

436 frame_size,

437 max_slice_nums=self.get_video_max_slice_num(),

436 - use_image_id=False,

438 )

439

440 def get_max_video_tokens(

@@ -539,14 +540,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

540 use_image_id=False,

541 ) * num_frames

542

542 - def get_embed_is_patch(

543 - self,

544 - input_ids: list[int],

545 - ) -> torch.Tensor:

546 - tokenizer = self.info.get_tokenizer()

547 - unk_token_id = tokenizer.get_vocab()["<unk>"]

548 - return torch.tensor(input_ids) == unk_token_id

549 -

543 def process_images(

544 self,

545 mm_data: Mapping[str, object],

@@ -570,26 +563,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

563 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

564 )

565

573 - image_sizes = [

574 - parsed_images.get_image_size(i) for i in range(len(parsed_images))

575 - ]

576 - image_repl_features = [

577 - self.get_image_prompt_texts(size, idx)

578 - for idx, size in enumerate(image_sizes)

579 - ]

580 -

566 tokenizer = self.info.get_tokenizer()

582 - image_repls_feature_tokens = [

583 - tokenizer.encode(image_repl, add_special_tokens=False)

584 - for image_repl in image_repl_features

585 - ]

586 -

587 - embed_is_patch = [

588 - self.get_embed_is_patch(image_repl_tokens)

589 - for image_repl_tokens in image_repls_feature_tokens

590 - ]

591 - image_inputs["embed_is_patch"] = embed_is_patch

592 -

567 unk_token_id = tokenizer.get_vocab()["<unk>"]

568 image_inputs["image_token_id"] = torch.tensor(unk_token_id)

569

@@ -625,31 +599,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

599 out_keys={"pixel_values", "image_sizes", "tgt_sizes"},

600 )

601

628 - frame_sizes = [

629 - parsed_videos.get_frame_size(i) for i in range(len(parsed_videos))

630 - ]

631 - num_frames = [

632 - parsed_videos.get_num_frames(i) for i in range(len(parsed_videos))

633 - ]

634 - video_repl_features = [

635 - self.get_video_prompt_texts(size, nframes)

636 - for size, nframes in zip(frame_sizes, num_frames)

637 - ]

638 -

639 - tokenizer = self.info.get_tokenizer()

640 - video_repls_feature_tokens = [

641 - tokenizer.encode(video_repl, add_special_tokens=False)

642 - for video_repl in video_repl_features

643 - ]

644 -

645 - embed_is_patch = [

646 - self.get_embed_is_patch(video_repl_tokens)

647 - for video_repl_tokens in video_repls_feature_tokens

648 - ]

649 - video_inputs["embed_is_patch"] = embed_is_patch

650 -

602 video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}

603

604 + tokenizer = self.info.get_tokenizer()

605 unk_token_id = tokenizer.get_vocab()["<unk>"]

606 video_inputs["video_token_id"] = torch.tensor(unk_token_id)

607

@@ -740,7 +692,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

692

693 image_size = images.get_image_size(item_idx)

694

743 - return self.get_image_prompt_texts(image_size, item_idx)

695 + return PromptUpdateDetails.select_text(

696 + self.get_image_prompt_texts(image_size, item_idx),

697 + "<unk>",

698 + )

699

700 def get_video_replacement(item_idx: int):

701 videos = mm_items.get_items(

@@ -749,7 +704,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

704 frame_size = videos.get_frame_size(item_idx)

705 num_frames = videos.get_num_frames(item_idx)

706

752 - return self.get_video_prompt_texts(frame_size, num_frames)

707 + return PromptUpdateDetails.select_text(

708 + self.get_video_prompt_texts(frame_size, num_frames),

709 + "<unk>",

710 + )

711

712 get_replacement = {

713 "image": get_image_replacement,

@@ -832,14 +790,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

790 assert isinstance(image_token_id, torch.Tensor)

791 self.mm_token_ids.add(image_token_id.flatten().unique().item())

792

835 - embed_is_patch = kwargs.pop("embed_is_patch")

836 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

837 - raise ValueError(

838 - f"Incorrect type of embed_is_patch for {modality=}. "

839 - f"Got type: {type(embed_is_patch)}")

840 -

841 - embed_is_patch = flatten_bn(embed_is_patch)

842 -

793 if image_embeds is not None:

794 if not isinstance(image_embeds, (torch.Tensor, list)):

795 raise ValueError(

@@ -851,7 +801,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

801 return MiniCPMVImageEmbeddingInputs(

802 type="image_embeds",

803 image_embeds=image_embeds_flat,

854 - embed_is_patch=embed_is_patch,

804 )

805

806 if not isinstance(pixel_values, (torch.Tensor, list)):

@@ -879,7 +828,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

828 type="pixel_values",

829 pixel_values=pixel_values_flat,

830 tgt_sizes=tgt_sizes_flat,

882 - embed_is_patch=embed_is_patch,

831 num_slices=num_slices_flat,

832 )

833

@@ -936,19 +884,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

884 if modality == "images":

885 image_input = modalities["images"]

886 image_features = self._process_vision_input(image_input)

939 - multimodal_embeddings += tuple(

940 - scatter_patch_features(

941 - image_features,

942 - image_input["embed_is_patch"],

943 - ))

887 + multimodal_embeddings += tuple(image_features)

888 if modality == "videos":

889 video_input = modalities["videos"]

890 video_features = self._process_vision_input(video_input)

947 - multimodal_embeddings += tuple(

948 - scatter_patch_features(

949 - video_features,

950 - video_input["embed_is_patch"],

951 - ))

891 + multimodal_embeddings += tuple(video_features)

892

893 return multimodal_embeddings

894

@@ -971,7 +911,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

911 inputs_embeds = merge_multimodal_embeddings(

912 input_ids,

913 inputs_embeds,

974 - select_patch_features(multimodal_embeddings),

914 + multimodal_embeddings,

915 list(self.mm_token_ids),

916 )

917 return inputs_embeds

vllm/model_executor/models/mistral3.py

+6 -44

@@ -27,7 +27,8 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

27 MultiModalDataItems)

28 from vllm.multimodal.processing import (BaseMultiModalProcessor,

29 BaseProcessingInfo, ProcessingCache,

30 - PromptReplacement, PromptUpdate)

30 + PromptReplacement, PromptUpdate,

31 + PromptUpdateDetails)

32 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

33 from vllm.sequence import IntermediateTensors

@@ -35,8 +36,7 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

36 from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel

37 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

38 maybe_prefix, merge_multimodal_embeddings)

38 -from .vision import (get_vision_encoder_info, scatter_patch_features,

39 - select_patch_features)

39 +from .vision import get_vision_encoder_info

42 class Mistral3ImagePixelInputs(TypedDict):

@@ -49,14 +49,6 @@ class Mistral3ImagePixelInputs(TypedDict):

49 in which case the data is passed as a list instead of a batched tensor.

50 """

52 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

53 - """

54 - A boolean mask indicating which image embeddings correspond

55 - to patch tokens.

56 -

57 - Shape: `(batch_size, num_images, num_embeds)`

58 - """

59 -

53 class Mistral3PatchMerger(nn.Module):

54 """

@@ -266,23 +258,6 @@ class Mistral3MultiModalProcessor(

258 p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)

259 ]

260

269 - hf_config = self.info.get_hf_config()

270 - vision_config = hf_config.vision_config

271 - assert isinstance(vision_config, PixtralVisionConfig)

272 - encoder_info = PixtralHFEncoderInfo(vision_config)

273 -

274 - tile_sizes = [

275 - encoder_info.get_patch_grid_size(

276 - image_width=pixel_value.shape[-1],

277 - image_height=pixel_value.shape[-2],

278 - ) for pixel_value in processed_outputs["pixel_values"]

279 - ]

280 - embed_is_patch = [

281 - torch.tensor(([True] * ncols + [False]) * nrows)

282 - for ncols, nrows in tile_sizes

283 - ]

284 - processed_outputs["embed_is_patch"] = embed_is_patch

285 -

261 return processed_outputs

262

263 def _get_mm_fields_config(

@@ -292,7 +267,6 @@ class Mistral3MultiModalProcessor(

267 ) -> Mapping[str, MultiModalFieldConfig]:

268 return dict(

269 pixel_values=MultiModalFieldConfig.batched("image"),

295 - embed_is_patch=MultiModalFieldConfig.batched("image"),

270 image_embeds=MultiModalFieldConfig.batched("image"),

271 )

272

@@ -327,7 +301,7 @@ class Mistral3MultiModalProcessor(

301 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows

302 tokens[-1] = image_end_id

303

330 - return tokens

304 + return PromptUpdateDetails.select_token_id(tokens, image_token_id)

305

306 return [

307 PromptReplacement(

@@ -418,8 +392,6 @@ def init_vision_tower_for_llava(

392 )

393

394

421 -# TODO(mgoin): Support V1, there are issues with image batching/chunking

422 -# that need to be resolved first.

395 @MULTIMODAL_REGISTRY.register_processor(

396 _build_mistral3_processor,

397 info=_build_mistral3_info,

@@ -509,16 +481,9 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

481 raise ValueError("Incorrect type of pixel values. "

482 f"Got type: {type(pixel_values)}")

483

512 - assert self.config.vision_config.model_type == "pixtral"

513 - embed_is_patch = kwargs.pop("embed_is_patch")

514 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

515 - raise ValueError("Incorrect type of embed_is_patch. "

516 - f"Got type: {type(embed_is_patch)}")

517 -

484 return Mistral3ImagePixelInputs(

485 type="pixel_values_pixtral",

486 pixel_values=flatten_bn(pixel_values),

521 - embed_is_patch=flatten_bn(embed_is_patch),

487 )

488

489 def _process_image_input(

@@ -557,10 +522,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

522

523 vision_embeddings = self._process_image_input(image_input)

524

560 - return scatter_patch_features(

561 - vision_embeddings,

562 - image_input["embed_is_patch"],

563 - )

525 + return vision_embeddings

526

527 def get_input_embeddings(

528 self,

@@ -572,7 +534,7 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsMultiModal,

534 inputs_embeds = merge_multimodal_embeddings(

535 input_ids,

536 inputs_embeds,

575 - select_patch_features(multimodal_embeddings),

537 + multimodal_embeddings,

538 self.config.image_token_index,

539 )

540 return inputs_embeds

vllm/model_executor/models/mllama4.py

+33 -66

@@ -33,7 +33,6 @@ from vllm.attention.layer import MultiHeadAttention

33 from vllm.config import VllmConfig

34 from vllm.distributed import get_tensor_model_parallel_world_size

35 from vllm.inputs import InputProcessingContext

36 -from vllm.logger import init_logger

36 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

37 QKVParallelLinear,

38 RowParallelLinear)

@@ -50,7 +49,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

49 MultiModalDataItems)

50 from vllm.multimodal.processing import (BaseMultiModalProcessor,

51 BaseProcessingInfo, PromptReplacement,

53 - PromptUpdate)

52 + PromptUpdate, PromptUpdateDetails)

53 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

54 from vllm.sequence import IntermediateTensors

@@ -58,9 +57,6 @@ from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

57 from .llama4 import Llama4ForCausalLM

58 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,

59 merge_multimodal_embeddings)

61 -from .vision import scatter_patch_features, select_patch_features

62 -

63 -logger = init_logger(__name__)

62 class Llama4ImagePatchInputs(TypedDict):

@@ -77,11 +73,7 @@ class Llama4ImagePatchInputs(TypedDict):

73 This is used to split the embeddings which has the first two dimensions

74 flattened just like `flat_data`.

75 """

80 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

81 - """

82 - A boolean mask indicating which image embeddings correspond

83 - to patch tokens.

84 - """

76 +

77 aspect_ratios: Union[torch.Tensor, list[torch.Tensor]]

78 """

79 A list of aspect ratios corresponding to the number of tiles

@@ -510,11 +502,10 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):

502 mm_counts: Mapping[str, int],

503 ) -> Mapping[str, int]:

504 vision_config = self.get_hf_config().vision_config

513 - # image_start + local tiles * (patches + 1 x separator) +

514 - # 1 global tile * (image x 1 + patches) + image_end

515 - token_per_chunk = self.get_patch_per_chunk(vision_config) + 1

516 - mm_max_tokens = (self.get_max_num_tiles() + 1) * token_per_chunk + 2

517 - return {"image": mm_max_tokens}

505 + patch_per_chunk = self.get_patch_per_chunk(vision_config)

506 + num_patches = self.get_max_num_tiles() + 1

507 +

508 + return {"image": patch_per_chunk * num_patches}

509

510 def get_image_size_with_most_features(self) -> ImageSize:

511 vision_config = self.get_hf_config().vision_config

@@ -523,6 +514,14 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):

514 return ImageSize(height=self.get_max_num_tiles() * image_size,

515 width=image_size)

516

517 + def get_max_image_tokens(self) -> int:

518 + target_width, target_height = self.get_image_size_with_most_features()

519 +

520 + return self.get_num_image_tokens(

521 + image_width=target_width,

522 + image_height=target_height,

523 + )

524 +

525

526 class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

527 ):

@@ -578,33 +577,9 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

577 for (r_h, r_w) in aspect_ratios

578 ]

579

581 - # embed_is_patch should have one feature per image-related token:

582 - # <|image_start|>, <|tile_*_separator|>, <|image|>, <|image_end|>

583 - # -> False

584 - # <|patch|> -> True

585 - # embed_is_patch has no entries corresponding to non-image-related

586 - # tokens.

587 - patch_id = tokenizer.get_vocab()[processor.img_patch_token]

588 - num_patches_per_chunk = self.info.get_patch_per_chunk(

589 - vision_config)

590 - expanded_image_tokens_list = [

591 - processor._prompt_split_image(aspect_ratio,

592 - num_patches_per_chunk)

593 - for aspect_ratio in aspect_ratios

594 - ]

595 - expanded_image_token_ids = [

596 - tokenizer.encode(image_tokens, add_special_tokens=False)

597 - for image_tokens in expanded_image_tokens_list

598 - ]

599 - embed_is_patch = [

600 - torch.tensor(tokens) == patch_id

601 - for tokens in expanded_image_token_ids

602 - ]

603 -

580 processed_outputs["aspect_ratios"] = aspect_ratios

581 processed_outputs["patches_per_image"] = torch.tensor(

582 patches_per_image)

607 - processed_outputs["embed_is_patch"] = embed_is_patch

583

584 return processed_outputs

585

@@ -619,7 +594,6 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

594 "image", patches_per_image),

595 patches_per_image=MultiModalFieldConfig.batched("image"),

596 aspect_ratios=MultiModalFieldConfig.batched("image"),

622 - embed_is_patch=MultiModalFieldConfig.batched("image"),

597 )

598

599 def _get_prompt_updates(

@@ -639,12 +613,17 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]

613 num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)

614 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

615 image_token = hf_processor.image_token

616 + img_patch_token = hf_processor.img_patch_token

617

618 def get_replacement(item_idx: int):

619 aspect_ratio = out_mm_kwargs["aspect_ratios"][item_idx]

645 - return hf_processor._prompt_split_image(

620 +

621 + repl = hf_processor._prompt_split_image(

622 aspect_ratio=aspect_ratio,

647 - num_patches_per_chunk=num_patches_per_chunk)

623 + num_patches_per_chunk=num_patches_per_chunk,

624 + )

625 +

626 + return PromptUpdateDetails.select_text(repl, img_patch_token)

627

628 return [

629 PromptReplacement(

@@ -737,11 +716,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

716 flat_pixel_values = flatten_bn(pixel_values, concat=True)

717 patches_per_image = flatten_bn(kwargs.pop("patches_per_image"))

718

740 - embed_is_patch = kwargs.pop("embed_is_patch", None)

741 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

742 - raise ValueError("Incorrect type of embed_is_patch. "

743 - f"Got type: {type(embed_is_patch)}")

744 -

719 aspect_ratios = kwargs.pop("aspect_ratios", None)

720 if not isinstance(aspect_ratios, (torch.Tensor, list)):

721 raise ValueError("Incorrect type of aspect_ratios. "

@@ -751,7 +725,6 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

725 type="pixel_values",

726 flat_data=flat_pixel_values,

727 patches_per_image=patches_per_image,

754 - embed_is_patch=embed_is_patch,

728 aspect_ratios=aspect_ratios,

729 )

730

@@ -759,10 +732,15 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

732 self, image_input: Llama4ImagePatchInputs) -> MultiModalEmbeddings:

733 flat_data = image_input["flat_data"]

734 patches_per_image = image_input["patches_per_image"].tolist()

735 +

736 vision_embeddings_flat = self.vision_model(flat_data)

737 vision_embeddings_flat = self.multi_modal_projector(

738 vision_embeddings_flat)

765 - return vision_embeddings_flat.split(patches_per_image, dim=0)

739 +

740 + return [

741 + img.flatten(0, 1)

742 + for img in vision_embeddings_flat.split(patches_per_image, dim=0)

743 + ]

744

745 def get_multimodal_embeddings(self,

746 **kwargs) -> Optional[MultiModalEmbeddings]:

@@ -770,20 +748,7 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

748 if image_input is None:

749 return None

750

773 - # num_images x [num_chunks, num_patches, hidden_dim]

774 - image_features = self._process_image_input(image_input)

775 - # num_images x [num_chunks x num_patches, hidden_dim]

776 - image_features_flat = [img.flatten(0, 1) for img in image_features]

777 - # num_images x [1, input_len] -> num_images x [input_len]

778 - embed_is_patch_flat = [

779 - is_patch.flatten(0, 1)

780 - for is_patch in image_input["embed_is_patch"]

781 - ]

782 -

783 - return scatter_patch_features(

784 - image_features_flat,

785 - embed_is_patch_flat,

786 - )

751 + return self._process_image_input(image_input)

752

753 def get_input_embeddings(

754 self,

@@ -794,9 +759,11 @@ class Llama4ForConditionalGeneration(nn.Module, SupportsMultiModal,

759

760 if multimodal_embeddings is not None:

761 inputs_embeds = merge_multimodal_embeddings(

797 - input_ids, inputs_embeds,

798 - select_patch_features(multimodal_embeddings),

799 - self.config.image_token_index)

762 + input_ids,

763 + inputs_embeds,

764 + multimodal_embeddings,

765 + self.config.image_token_index,

766 + )

767

768 return inputs_embeds

769

vllm/model_executor/models/molmo.py

+13 -61

@@ -46,7 +46,8 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

46 MultiModalDataItems)

47 from vllm.multimodal.processing import (BaseMultiModalProcessor,

48 BaseProcessingInfo, PromptIndexTargets,

49 - PromptInsertion, PromptUpdate)

49 + PromptInsertion, PromptUpdate,

50 + PromptUpdateDetails)

51 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

52 from vllm.sequence import IntermediateTensors

@@ -56,7 +57,6 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

57 is_pp_missing_parameter,

58 make_empty_intermediate_tensors_factory, make_layers,

59 maybe_prefix, merge_multimodal_embeddings)

59 -from .vision import scatter_patch_features, select_patch_features

61 # TODO: hard-coded for now. Consider making it configurable.

62 VIT_LAYERS = [-2, -9]

@@ -84,14 +84,6 @@ class MolmoImageInputs(TypedDict):

84 Shape: `(batch_size * num_images, num_crops, num_patch)`

85 """

87 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

88 - """

89 - A boolean mask indicating which image embeddings correspond

90 - to patch tokens.

91 -

92 - Shape: `(batch_size * num_images, num_embeds)`

93 - """

94 -

87 num_crops: torch.Tensor

88 """Shape: `(batch_size * num_images)`"""

@@ -1146,30 +1138,6 @@ class MolmoProcessorWrapper:

1138 if image_input_idx is not None:

1139 feat_is_patch = image_input_idx >= 0

1140

1149 - input_is_embed = torch.isin(

1150 - input_ids,

1151 - torch.tensor([

1152 - self.image_patch_id,

1153 - self.im_col_id,

1154 - self.im_start_id,

1155 - self.im_end_id,

1156 - ]),

1157 - )

1158 - embed_ids = input_ids[input_is_embed]

1159 - embed_is_patch = embed_ids == self.image_patch_id

1160 - assert embed_is_patch.sum() == feat_is_patch.sum()

1161 -

1162 - # image_tokens = extra_joint + joint

1163 - # Both `extra_joint` and `joint` have `im_start_id` and `im_end_id`

1164 - embed_start = torch.nonzero(embed_ids == self.im_start_id)[::2, 0]

1165 - embed_end = torch.nonzero(embed_ids == self.im_end_id)[1::2, 0]

1166 - assert len(embed_start) == len(embed_end) == len(images)

1167 -

1168 - embed_is_patch = [

1169 - embed_is_patch[start:end + 1]

1170 - for start, end in zip(embed_start, embed_end)

1171 - ]

1172 -

1141 tilings = [

1142 self.select_tiling(

1143 image_width=image.size[0],

@@ -1181,7 +1149,6 @@ class MolmoProcessorWrapper:

1149 assert num_crops.sum() == len(feat_is_patch)

1150

1151 outputs["feat_is_patch"] = feat_is_patch

1184 - outputs["embed_is_patch"] = embed_is_patch

1152 outputs["num_crops"] = num_crops

1153 outputs["img_patch_id"] = self.image_patch_id

1154

@@ -1220,17 +1187,13 @@ class MolmoProcessingInfo(BaseProcessingInfo):

1187 )

1188 pooling_size = processor.pooling_size

1189

1223 - base_image_input_size = processor.base_image_input_size

1224 - base_image_input_d = processor.image_patch_size

1225 -

1226 - crop_patches = base_image_input_size[0] // base_image_input_d

1190 + image_token_length_w = processor.image_token_length_w

1191 + image_token_length_h = processor.image_token_length_h

1192

1228 - per_row = ncols // pooling_size + 1

1229 - joint = per_row * (nrows // pooling_size) + 2

1230 - image_token_length = (crop_patches + pooling_size - 1) // pooling_size

1231 - resize = (image_token_length + 1) * image_token_length + 2

1193 + extra = image_token_length_w * image_token_length_h

1194 + joint = ((ncols + 1) // pooling_size) * ((nrows + 1) // pooling_size)

1195

1233 - return resize + joint

1196 + return extra + joint

1197

1198 def get_max_image_tokens(self) -> int:

1199 target_width, target_height = self.get_image_size_with_most_features()

@@ -1328,7 +1291,6 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1291 "image", num_crops),

1292 feat_is_patch=MultiModalFieldConfig.flat_from_sizes(

1293 "image", num_crops),

1331 - embed_is_patch=MultiModalFieldConfig.batched("image"),

1294 num_crops=MultiModalFieldConfig.batched("image"),

1295 img_patch_id=MultiModalFieldConfig.shared("image", num_images),

1296 )

@@ -1368,8 +1330,10 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):

1330 joint = ([img_start_id] + joint_row *

1331 ((nrows + 1) // pooling_size) + [img_end_id])

1332

1371 - image_tokens = extra_joint + joint

1372 - return image_tokens

1333 + return PromptUpdateDetails.select_token_id(

1334 + extra_joint + joint,

1335 + embed_token_id=img_patch_id,

1336 + )

1337

1338 return [

1339 PromptInsertion(

@@ -1475,11 +1439,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1439 raise ValueError("Incorrect type of feat_is_patch. "

1440 f"Got type: {type(feat_is_patch)}")

1441

1478 - embed_is_patch = kwargs.pop("embed_is_patch", None)

1479 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

1480 - raise ValueError("Incorrect type of embed_is_patch. "

1481 - f"Got type: {type(embed_is_patch)}")

1482 -

1442 num_crops = kwargs.pop("num_crops", None)

1443 if not isinstance(num_crops, (torch.Tensor, list)):

1444 raise ValueError("Incorrect type of num_crops. "

@@ -1491,14 +1450,12 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1450 f"Got type: {type(img_patch_id)}")

1451 self.img_patch_id = img_patch_id.flatten().unique().item()

1452

1494 - embed_is_patch = flatten_bn(embed_is_patch)

1453 num_crops = flatten_bn(num_crops, concat=True)

1454

1455 return MolmoImageInputs(

1456 images=images,

1457 image_masks=image_masks,

1458 feat_is_patch=feat_is_patch,

1501 - embed_is_patch=embed_is_patch,

1459 num_crops=num_crops,

1460 )

1461

@@ -1537,12 +1494,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1494 if image_input is None:

1495 return None

1496

1540 - image_features = self._process_image_input(image_input)

1541 -

1542 - return scatter_patch_features(

1543 - image_features,

1544 - image_input["embed_is_patch"],

1545 - )

1497 + return self._process_image_input(image_input)

1498

1499 def get_input_embeddings(

1500 self,

@@ -1556,7 +1508,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1508 inputs_embeds = merge_multimodal_embeddings(

1509 input_ids,

1510 inputs_embeds,

1559 - select_patch_features(multimodal_embeddings),

1511 + multimodal_embeddings,

1512 self.img_patch_id,

1513 )

1514 return inputs_embeds

vllm/model_executor/models/nvlm_d.py

+2 -30

@@ -57,7 +57,7 @@ class NVLMProcessor(BaseInternVLProcessor):

57 # when trying to find "<tile" as a subsequence of "<Image><tile"

58 repl = "<Image>" + features + "</Image>"

60 - return PromptUpdateDetails(full=repl, features=repl)

60 + return PromptUpdateDetails.select_text(repl, IMG_PAD)

63 class NVLMProcessingInfo(BaseInternVLProcessingInfo):

@@ -84,31 +84,6 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):

84 **kwargs,

85 )

87 - def get_max_image_tokens(self) -> int:

88 - hf_processor = self.get_hf_processor()

89 - tokenizer = hf_processor.tokenizer

90 -

91 - max_num_patches = hf_processor.max_dynamic_patch

92 - # we need +1 here because max_dynamic_patch in config doesn't

93 - # include the thumbnail patch

94 - tile_pos_identifiers = [

95 - f"<tile_{i+1}>" for i in range(max_num_patches)

96 - ]

97 - if hf_processor.use_thumbnail and max_num_patches != 1:

98 - tile_pos_identifiers += ["<tile_global_thumbnail>"]

99 -

100 - # "<Image><tile" is tokenized as ["<Image", "><", "tile"]

101 - # so we include <tile_1> in the start_str

102 - start_str = "<Image>" + tile_pos_identifiers.pop(0)

103 - end_str = "</Image>"

104 - start_token_len = len(tokenizer.encode(start_str))

105 - end_token_len = len(tokenizer.encode(end_str))

106 - tile_token_len = sum(

107 - len(tokenizer.encode(identifier))

108 - for identifier in tile_pos_identifiers)

109 - non_image_tokens_num = start_token_len + end_token_len + tile_token_len

110 - return super().get_max_image_tokens() + non_image_tokens_num

111 -

88 class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):

@@ -177,10 +152,7 @@ class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):

152

153 repl = hf_processor.get_image_repl(feature_size, num_patches)

154

180 - return PromptUpdateDetails(

181 - full=repl.full + "\n",

182 - features=repl.features + "\n",

183 - )

155 + return PromptUpdateDetails.select_text(repl.full + "\n", IMG_PAD)

156

157 # See note in dummy data regarding why we have the extra newline

158 return [

vllm/model_executor/models/paligemma.py

+3 -3

@@ -162,9 +162,9 @@ class PaliGemmaMultiModalProcessor(

162 modality="image",

163 target=PromptIndexTargets.prefix(

164 [bos_token_id] if tokenizer.add_bos_token else []),

165 - insertion=PromptUpdateDetails(

166 - full=image_tokens + [bos_token_id],

167 - features=image_tokens,

165 + insertion=PromptUpdateDetails.select_token_id(

166 + image_tokens + [bos_token_id],

167 + embed_token_id=image_token_id,

168 ),

169 )

170 ]

vllm/model_executor/models/phi3v.py

+3 -8

@@ -40,8 +40,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

40 from vllm.multimodal.processing import (BaseMultiModalProcessor,

41 BaseProcessingInfo, BoundPromptUpdate,

42 PlaceholderFeaturesInfo,

43 - PromptReplacement, PromptUpdate,

44 - PromptUpdateDetails)

43 + PromptReplacement, PromptUpdate)

44 # yapf: enable

45 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

46 from vllm.sequence import IntermediateTensors

@@ -443,12 +442,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

442 processor=hf_processor,

443 )

444

446 - image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens

447 -

448 - return PromptUpdateDetails(

449 - full=image_tokens,

450 - features=image_tokens,

451 - )

445 + return [_IMAGE_TOKEN_ID] * num_image_tokens

446

447 num_images = mm_items.get_count("image", strict=False)

448

@@ -517,6 +511,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):

511 item_idx=p.item_idx,

512 start_idx=p.start_idx - 1,

513 tokens=p.tokens,

514 + is_embed=p.is_embed,

515 ) for p in ps

516 ]

517 for modality, ps in placeholders.items()

vllm/model_executor/models/pixtral.py

+8 -40

@@ -37,7 +37,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

37 MultiModalDataItems)

38 from vllm.multimodal.processing import (BaseMultiModalProcessor,

39 BaseProcessingInfo, PromptReplacement,

40 - PromptUpdate)

40 + PromptUpdate, PromptUpdateDetails)

41 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

42 from vllm.sequence import IntermediateTensors

43 from vllm.transformers_utils.tokenizer import (MistralTokenizer,

@@ -46,8 +46,7 @@ from vllm.transformers_utils.tokenizer import (MistralTokenizer,

46 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

47 from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,

48 merge_multimodal_embeddings)

49 -from .vision import (VisionEncoderInfo, resolve_visual_encoder_outputs,

50 - scatter_patch_features, select_patch_features)

49 +from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs

51 try:

52 from xformers import ops as xops

@@ -68,14 +67,6 @@ class PixtralImagePixelInputs(TypedDict):

67 The result of stacking :attr:`ImageEncoding.tokens` from each prompt.

68 """

71 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

72 - """

73 - A boolean mask indicating which image embeddings correspond

74 - to patch tokens.

75 -

76 - Shape: `(batch_size * num_images, num_embeds)`

77 - """

78 -

71 class PixtralProcessorAdapter:

72 """

@@ -144,11 +135,8 @@ class PixtralProcessorAdapter:

135 "For more info, see: "

136 "https://github.com/vllm-project/vllm/issues/8411.")

137

147 - image_token_id = self.image_token_id

148 -

138 images_processed = list[torch.Tensor]()

139 images_tokens = list[torch.Tensor]()

151 - images_embed_is_patch = list[torch.Tensor]()

140

141 for image in images:

142 image_inputs = self.image_processor(ImageChunk(image=image))

@@ -157,12 +145,10 @@ class PixtralProcessorAdapter:

145

146 images_processed.append(image_processed)

147 images_tokens.append(image_tokens)

160 - images_embed_is_patch.append(image_tokens == image_token_id)

148

149 return {

150 "input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),

151 "images": images_processed,

165 - "embed_is_patch": images_embed_is_patch,

152 }

153

154

@@ -213,7 +199,7 @@ class PixtralProcessingInfo(BaseProcessingInfo):

199 ncols, nrows = processor.image_processor._image_to_num_tokens(

200 Image.new("RGB", (image_width, image_height)))

201

216 - return (ncols + 1) * nrows

202 + return ncols * nrows

203

204 def get_image_size_with_most_features(self) -> ImageSize:

205 image_processor = self.get_hf_processor().image_processor

@@ -263,10 +249,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

249 hf_inputs: Mapping[str, NestedTensors],

250 hf_processor_mm_kwargs: Mapping[str, object],

251 ) -> Mapping[str, MultiModalFieldConfig]:

266 - return dict(

267 - images=MultiModalFieldConfig.batched("image"),

268 - embed_is_patch=MultiModalFieldConfig.batched("image"),

269 - )

252 + return dict(images=MultiModalFieldConfig.batched("image"))

253

254 def _get_prompt_updates(

255 self,

@@ -290,7 +273,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

273 tokens = ([image_token_id] * ncols + [image_break_id]) * nrows

274 tokens[-1] = image_end_id

275

293 - return tokens

276 + return PromptUpdateDetails.select_token_id(tokens, image_token_id)

277

278 return [

279 PromptReplacement(

@@ -381,17 +364,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

364 raise ValueError("Incorrect type of images. "

365 f"Got type: {type(images)}")

366

384 - embed_is_patch = kwargs.pop("embed_is_patch")

385 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

386 - raise ValueError("Incorrect type of embed_is_patch. "

387 - f"Got type: {type(embed_is_patch)}")

388 -

389 - embed_is_patch = flatten_bn(embed_is_patch)

390 -

367 return PixtralImagePixelInputs(

368 type="pixel_values",

369 images=flatten_bn(images),

394 - embed_is_patch=embed_is_patch,

370 )

371

372 def _process_image_input(

@@ -427,12 +402,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

402 if image_input is None:

403 return None

404

430 - image_features = self._process_image_input(image_input)

431 -

432 - return scatter_patch_features(

433 - image_features,

434 - image_input["embed_is_patch"],

435 - )

405 + return self._process_image_input(image_input)

406

407 def get_input_embeddings(

408 self,

@@ -444,7 +414,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

414 inputs_embeds = merge_multimodal_embeddings(

415 input_ids,

416 inputs_embeds,

447 - select_patch_features(multimodal_embeddings),

417 + multimodal_embeddings,

418 self.vision_args.image_token_id,

419 )

420 return inputs_embeds

@@ -963,9 +933,7 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):

933 image_width=image_width,

934 image_height=image_height,

935 )

966 -

967 - # Consider the image_break_token

968 - return (ncols + 1) * nrows

936 + return ncols * nrows

937

938 def get_max_image_tokens(self) -> int:

939 image_size = self.get_image_size()

vllm/model_executor/models/qwen2_audio.py

+3 -3

@@ -229,9 +229,9 @@ class Qwen2AudioMultiModalProcessor(

229

230 audio_tokens = [audio_token_id] * num_features

231

232 - return PromptUpdateDetails(

233 - full=[audio_bos_id] + audio_tokens + [audio_eos_id],

234 - features=audio_tokens,

232 + return PromptUpdateDetails.select_token_id(

233 + [audio_bos_id] + audio_tokens + [audio_eos_id],

234 + embed_token_id=audio_token_id,

235 )

236

237 return [

vllm/model_executor/models/qwen_vl.py

+3 -3

@@ -647,9 +647,9 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):

647 PromptReplacement(

648 modality="image",

649 target=[img_start_id, img_end_id],

650 - replacement=PromptUpdateDetails(

651 - full=[img_start_id] + image_tokens + [img_end_id],

652 - features=image_tokens,

650 + replacement=PromptUpdateDetails.select_token_id(

651 + [img_start_id] + image_tokens + [img_end_id],

652 + embed_token_id=img_pad_id,

653 ),

654 )

655 ]

vllm/model_executor/models/skyworkr1v.py

+3 -39

@@ -40,7 +40,6 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer

40 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

41 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

42 maybe_prefix, merge_multimodal_embeddings)

43 -from .vision import scatter_patch_features, select_patch_features

44 IMG_START = '<img>'

45 IMG_END = '</img>'

@@ -61,14 +60,6 @@ class SkyworkR1VImagePixelInputs(TypedDict):

60 num_patches: torch.Tensor

61 """Shape: `(batch_size * num_images)`"""

64 - embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

65 - """

66 - A boolean mask indicating which image embeddings correspond

67 - to patch tokens.

68 -

69 - Shape: `(batch_size * num_images, num_embeds)`

70 - """

71 -

64 class SkyworkR1VImageEmbeddingInputs(TypedDict):

65 type: Literal["image_embeds"]

@@ -419,24 +410,13 @@ class BaseSkyworkR1VProcessor(ABC):

410 torch.tensor([len(item) for item in pixel_values_lst]),

411 }

412

422 - tokenizer = self.tokenizer

423 - image_token_id = self.image_token_id

424 -

425 - embed_is_patch = list[torch.Tensor]()

426 -

413 for pixel_values in pixel_values_lst:

414 num_patches = pixel_values.shape[0]

415 feature_size = num_patches * self.num_image_token

416

417 image_repl = self.get_image_repl(feature_size, num_patches)

432 - feature_tokens = tokenizer.encode(image_repl.features,

433 - add_special_tokens=False)

418

419 text = [t.replace('<image>', image_repl.full, 1) for t in text]

436 - embed_is_patch.append(

437 - torch.tensor(feature_tokens) == image_token_id)

438 -

439 - image_inputs["embed_is_patch"] = embed_is_patch

420

421 text_inputs = self.tokenizer(text)

422

@@ -460,7 +440,7 @@ class SkyworkR1VProcessor(BaseSkyworkR1VProcessor):

440 repl_features = IMG_CONTEXT * feature_size

441 repl_full = IMG_START + repl_features + IMG_END

442

463 - return PromptUpdateDetails(full=repl_full, features=repl_features)

443 + return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)

444

445

446 class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):

@@ -599,7 +579,6 @@ class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):

579 pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(

580 "image", image_num_patches),

581 image_num_patches=MultiModalFieldConfig.batched("image"),

602 - embed_is_patch=MultiModalFieldConfig.batched("image"),

582 image_embeds=MultiModalFieldConfig.batched("image"),

583 image_token_id=MultiModalFieldConfig.shared("image", num_images),

584 )

@@ -835,7 +814,6 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

814 self, **kwargs: object) -> Optional[SkyworkR1VImageInputs]:

815 pixel_values_flat = kwargs.pop("pixel_values_flat", None)

816 image_num_patches = kwargs.pop("image_num_patches", None)

838 - embed_is_patch = kwargs.pop("embed_is_patch", None)

817 image_embeds = kwargs.pop("image_embeds", None)

818

819 if pixel_values_flat is None and image_embeds is None:

@@ -864,20 +842,14 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

842 raise ValueError("Incorrect type of image_num_patches. "

843 f"Got type: {type(image_num_patches)}")

844

867 - if not isinstance(embed_is_patch, (torch.Tensor, list)):

868 - raise ValueError("Incorrect type of embed_is_patch. "

869 - f"Got type: {type(embed_is_patch)}")

870 -

845 pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)

846 image_num_patches = flatten_bn(image_num_patches, concat=True)

873 - embed_is_patch = flatten_bn(embed_is_patch)

847

848 return SkyworkR1VImagePixelInputs(

849 type="pixel_values",

850 pixel_values_flat=self._validate_pixel_values(

851 pixel_values_flat),

852 num_patches=image_num_patches,

880 - embed_is_patch=embed_is_patch,

853 )

854

855 raise AssertionError("This line should be unreachable.")

@@ -923,15 +895,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

895 if image_input is None:

896 return None

897

926 - image_features = self._process_image_input(image_input)

927 -

928 - if image_input["type"] != "pixel_values":

929 - return image_features

930 -

931 - return scatter_patch_features(

932 - image_features,

933 - image_input["embed_is_patch"],

934 - )

898 + return self._process_image_input(image_input)

899

900 def get_input_embeddings(

901 self,

@@ -945,7 +909,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):

909 inputs_embeds = merge_multimodal_embeddings(

910 input_ids,

911 inputs_embeds,

948 - select_patch_features(multimodal_embeddings),

912 + multimodal_embeddings,

913 self.img_context_token_id,

914 )

915 return inputs_embeds

vllm/model_executor/models/vision.py

+1 -76

@@ -1,8 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

3 from abc import ABC, abstractmethod

4 -from collections.abc import Sequence

5 -from typing import Final, Generic, Optional, Protocol, TypeVar, Union, cast

4 +from typing import Final, Generic, Optional, Protocol, TypeVar, Union

6 import torch

7 from transformers import PretrainedConfig

@@ -10,12 +9,9 @@ from transformers import PretrainedConfig

9 import vllm.envs as envs

10 from vllm.attention.selector import (backend_name_to_enum,

11 get_global_forced_attn_backend)

13 -from vllm.jsontree import JSONTree, json_map_leaves

12 from vllm.logger import init_logger

13 from vllm.platforms import _Backend, current_platform

17 -from .interfaces import MultiModalEmbeddings

18 -

15 logger = init_logger(__name__)

17 _C = TypeVar("_C", bound=PretrainedConfig)

@@ -155,74 +151,3 @@ def resolve_visual_encoder_outputs(

151 if post_layer_norm is not None and uses_last_layer:

152 hs_pool[-1] = post_layer_norm(encoder_outputs)

153 return torch.cat(hs_pool, dim=-1)

158 -

159 -

160 -def scatter_patch_features(

161 - patches: Union[torch.Tensor, Sequence[torch.Tensor]],

162 - embed_is_patch: Union[torch.Tensor, Sequence[torch.Tensor]],

163 -) -> tuple[torch.Tensor, ...]:

164 - """

165 - Scatter the patch features into a contiguous tensor that corresponds

166 - to the embedding tokens defined by the multimodal processor.

167 -

168 - The rest of the values in the tensor are set to NaN so that they

169 - can be filtered out by :func`select_patch_features`.

170 -

171 - Args:

172 - patches: The patch features for each image.

173 - Shape: `(num_images, <patch_dims>, feature_depth)`

174 - embed_is_patch: A boolean mask indicating which image embeddings

175 - correspond to patch tokens for each image.

176 - Shape: `(num_images, num_embeds)`

177 -

178 - Note:

179 - The original code only considers patch tokens as feature

180 - tokens, but our processor considers all image-related tokens

181 - as feature tokens because the feature tokens need to be

182 - consecutive in `input_ids`.

183 -

184 - Example:

185 - A simplified example for one image:

186 -

187 - .. code-block::

188 -

189 - Embedding tokens (from HF processor):

190 - [<start> <patch> <patch> <col> <patch> <patch> <col> <end> ]

191 -

192 - embed_is_patch (from HF processor):

193 - [ False True True False True True False False ]

194 -

195 - Encoder outputs (from model):

196 - [ p1 p2 p3 p4 ]

197 -

198 - The resulting embedding tensor is:

199 - [ nan p1 p2 nan p3 p4 nan nan ]

200 - """

201 - if len(patches) != len(embed_is_patch):

202 - raise ValueError(f"Inconsistent num_images: {len(patches)=} vs. "

203 - f"{len(embed_is_patch)=}")

204 -

205 - def get_embed_one(patches_one: torch.Tensor, e_is_patch: torch.Tensor):

206 - embed_one = patches_one.new_full(

207 - (e_is_patch.shape[0], patches_one.shape[-1]),

208 - fill_value=torch.nan,

209 - )

210 - embed_one[e_is_patch] = patches_one

211 - return embed_one

212 -

213 - return tuple(

214 - get_embed_one(patches_one, e_is_patch)

215 - for patches_one, e_is_patch in zip(patches, embed_is_patch))

216 -

217 -

218 -def select_patch_features(

219 - multimodal_embeddings: MultiModalEmbeddings) -> MultiModalEmbeddings:

220 - """

221 - Given the outputs of :func:`scatter_patch_features`, return only

222 - the values that correspond to patch features.

223 - """

224 - selected_features = json_map_leaves(

225 - lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),

226 - cast(JSONTree[torch.Tensor], multimodal_embeddings),

227 - )

228 - return cast(MultiModalEmbeddings, selected_features)

vllm/multimodal/base.py

+2 -2

@@ -385,8 +385,8 @@ class MultiModalPlaceholderMap:

385 for placeholder_dict, mm_item in zip(multi_modal_placeholders,

386 multi_modal_items):

387 placeholder = range(

388 - placeholder_dict["offset"],

389 - placeholder_dict["offset"] + placeholder_dict["length"],

388 + placeholder_dict.offset,

389 + placeholder_dict.offset + placeholder_dict.length,

390 )

391 intersection = range(

392 max(positions.start, placeholder.start),

vllm/multimodal/inputs.py

+29 -3

@@ -109,7 +109,8 @@ The built-in modalities are defined by :class:`MultiModalDataBuiltins`.

109 """

110

111

112 -class PlaceholderRange(TypedDict):

112 +@dataclass(frozen=True)

113 +class PlaceholderRange:

114 """

115 Placeholder location information for multi-modal data.

116

@@ -121,8 +122,8 @@ class PlaceholderRange(TypedDict):

122

123 .. code-block::

124

124 - A: { "offset": 0, "length": 4 }

125 - B: { "offset": 5, "length": 4 }

125 + A: PlaceholderRange(offset=0, length=4)

126 + B: PlaceholderRange(offset=5, length=4)

127 """

128

129 offset: int

@@ -131,6 +132,31 @@ class PlaceholderRange(TypedDict):

132 length: int

133 """The length of the placeholder."""

134

135 + is_embed: Optional[torch.Tensor] = None

136 + """

137 + A boolean mask of shape `(length,)` indicating which positions

138 + between `offset` and `offset + length` to assign embeddings to.

139 + """

140 +

141 + def get_num_embeds(self) -> int:

142 + if self.is_embed is None:

143 + return self.length

144 +

145 + return int(self.is_embed.sum().item())

146 +

147 + def __eq__(self, other: object) -> bool:

148 + if not isinstance(other, self.__class__):

149 + return False

150 + if not (self.offset, self.length) == (other.offset, other.length):

151 + return False

152 +

153 + if self.is_embed is None:

154 + return other.is_embed is None

155 + if other.is_embed is None:

156 + return self.is_embed is None

157 +

158 + return nested_tensors_equal(self.is_embed, other.is_embed)

159 +

160

161 NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,

162 tuple[torch.Tensor, ...]]

vllm/multimodal/processing.py

+52 -25

@@ -108,16 +108,46 @@ class PromptUpdateDetails(Generic[_S]):

108 full: _S

109 """The full content."""

110

111 - features: _S

111 + is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None

112 """

113 - The part of the content that corresponds to feature placeholders;

114 - this will be replaced by the output of the vision encoder during model

115 - inference.

113 + Given :attr:`full`, return a boolean mask of shape `(len(full),)`

114 + indicating which positions of `full` to assign embeddings to.

115 +

116 + `None` (default) means to assign embeddings to all positions of `full`.

117 +

118 + The embeddings are obtained by calling

119 + :class:`SupportsMultiModal.get_multimodal_embeddings`.

120 """

121

122 @staticmethod

123 def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":

120 - return PromptUpdateDetails(full=seq, features=seq)

124 + return PromptUpdateDetails(full=seq)

125 +

126 + @staticmethod

127 + def select_text(

128 + seq: _S,

129 + embed_text: str,

130 + ) -> "PromptUpdateDetails[_S]":

131 +

132 + def is_embed(full: "_BoundPromptSequence") -> torch.Tensor:

133 + embed_token_ids = encode_tokens(full.tokenizer, embed_text)

134 +

135 + return torch.isin(

136 + torch.tensor(full.token_ids),

137 + torch.tensor(embed_token_ids),

138 + )

139 +

140 + return PromptUpdateDetails(full=seq, is_embed=is_embed)

141 +

142 + @staticmethod

143 + def select_token_id(

144 + seq: _S,

145 + embed_token_id: int,

146 + ) -> "PromptUpdateDetails[_S]":

147 + return PromptUpdateDetails(

148 + full=seq,

149 + is_embed=lambda f: torch.tensor(f.token_ids) == embed_token_id,

150 + )

151

152

153 PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]

@@ -406,7 +436,7 @@ class _BoundPromptSequence:

436 @dataclass

437 class _BoundPromptContent:

438 full: _BoundPromptSequence

409 - features: _BoundPromptSequence

439 + is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]]

440

441

442 @dataclass

@@ -466,10 +496,8 @@ class BoundPromptUpdate:

496

497 bound_full = _BoundPromptSequence.from_seq(self.tokenizer,

498 content.full)

469 - bound_features = _BoundPromptSequence.from_seq(self.tokenizer,

470 - content.features)

499 bound_content = _BoundPromptContent(full=bound_full,

472 - features=bound_features)

500 + is_embed=content.is_embed)

501

502 if cache_key is not None:

503 self._content_cache[cache_key] = bound_content

@@ -605,15 +633,19 @@ class PlaceholderFeaturesInfo:

633 item_idx: int

634 start_idx: int

635 tokens: list[int]

636 + is_embed: Optional[torch.Tensor]

637

638 @property

639 def length(self) -> int:

640 return len(self.tokens)

641

642 def to_range(self) -> PlaceholderRange:

643 + # TODO: Is it worth it to optimize this by stripping the

644 + # leading and ending positions where `is_embed=False`?

645 return PlaceholderRange(

646 offset=self.start_idx,

647 length=self.length,

648 + is_embed=self.is_embed,

649 )

650

651

@@ -806,22 +838,17 @@ def _iter_placeholders(

838 continue

839

840 if prompt[start_idx:end_idx_full] == content_tokens_full:

809 - content_tokens_feat = content.features.token_ids

810 -

811 - try:

812 - match = next(

813 - iter_token_matches(content_tokens_full,

814 - content_tokens_feat))

815 - yield PlaceholderFeaturesInfo(

816 - modality=modality,

817 - item_idx=item_idx,

818 - start_idx=start_idx + match.start_idx,

819 - tokens=content_tokens_feat,

820 - )

821 - except StopIteration:

822 - raise AssertionError(

823 - f"{content_tokens_feat=} should be a "

824 - f"subsequence of {content_tokens_full=}") from None

841 + content_is_embed = content.is_embed

842 + if content_is_embed is not None:

843 + content_is_embed = content_is_embed(content.full)

844 +

845 + yield PlaceholderFeaturesInfo(

846 + modality=modality,

847 + item_idx=item_idx,

848 + start_idx=start_idx,

849 + tokens=content_tokens_full,

850 + is_embed=content_is_embed,

851 + )

852

853 # Exclude overlapping matches

854 start_idx = end_idx_full

vllm/multimodal/profiling.py

+1 -1

@@ -181,7 +181,7 @@ class MultiModalProfiler(Generic[_I]):

181 placeholders_by_modality = mm_inputs["mm_placeholders"]

182

183 total_placeholders_by_modality = {

184 - modality: sum(item["length"] for item in placeholders)

184 + modality: sum(item.get_num_embeds() for item in placeholders)

185 for modality, placeholders in placeholders_by_modality.items()

186 }

187 expected_placeholders_by_modality = {

vllm/multimodal/utils.py

+1 -1

@@ -340,7 +340,7 @@ def merge_and_sort_multimodal_metadata(

340 all_items.append((modality, placeholder, hash_value))

341

342 # Sort all items by offset

343 - all_items.sort(key=lambda x: x[1]['offset'])

343 + all_items.sort(key=lambda x: x[1].offset)

344

345 # Split into separate lists

346 sorted_modalities = [item[0] for item in all_items]

vllm/v1/core/kv_cache_utils.py

+3 -4

@@ -310,8 +310,7 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

310 # Note that we assume mm_positions is sorted by offset.

311 # We do not need to check all mm inputs if the start token index is out of

312 # range. This usually happens in the late prefill phase and decoding phase.

313 - if mm_positions[-1]["offset"] + mm_positions[-1][

314 - "length"] < start_token_idx:

313 + if mm_positions[-1].offset + mm_positions[-1].length < start_token_idx:

314 return extra_keys, start_mm_idx

315

316 # Support start_mm_idx == -1 to indicate the last mm input.

@@ -322,8 +321,8 @@ def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,

321 curr_mm_idx = start_mm_idx

322 while mm_positions and curr_mm_idx < len(mm_positions):

323 assert mm_hashes[curr_mm_idx] is not None

325 - offset = mm_positions[curr_mm_idx]["offset"]

326 - length = mm_positions[curr_mm_idx]["length"]

324 + offset = mm_positions[curr_mm_idx].offset

325 + length = mm_positions[curr_mm_idx].length

326 if end_token_idx > offset:

327 if start_token_idx > offset + length:

328 # This block has passed the current mm input.

vllm/v1/core/sched/scheduler.py

+4 -4

@@ -505,8 +505,8 @@ class Scheduler(SchedulerInterface):

505 assert mm_positions is not None

506 assert len(mm_positions) > 0

507 for i, pos_info in enumerate(mm_positions):

508 - start_pos = pos_info["offset"]

509 - num_encoder_tokens = pos_info["length"]

508 + start_pos = pos_info.offset

509 + num_encoder_tokens = pos_info.length

510

511 # The encoder output is needed if the two ranges overlap:

512 # [num_computed_tokens, num_computed_tokens + num_new_tokens) and

@@ -596,8 +596,8 @@ class Scheduler(SchedulerInterface):

596 if cached_encoder_input_ids:

597 for input_id in list(cached_encoder_input_ids):

598 mm_positions = request.mm_positions[input_id]

599 - start_pos = mm_positions["offset"]

600 - num_tokens = mm_positions["length"]

599 + start_pos = mm_positions.offset

600 + num_tokens = mm_positions.length

601 if start_pos + num_tokens <= request.num_computed_tokens:

602 # The encoder output is already processed and stored

603 # in the decoder's KV cache.

vllm/v1/request.py

+1 -1

@@ -121,7 +121,7 @@ class Request:

121

122 def get_num_encoder_tokens(self, input_id: int) -> int:

123 assert input_id < len(self.mm_positions)

124 - num_tokens = self.mm_positions[input_id]["length"]

124 + num_tokens = self.mm_positions[input_id].length

125 return num_tokens

126

127 @property

vllm/v1/worker/gpu_model_runner.py

+40 -21

@@ -19,7 +19,8 @@ from vllm.logger import init_logger

19 from vllm.model_executor.layers.fused_moe import FusedMoE

20 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding

21 from vllm.model_executor.model_loader import get_model

22 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

22 +from vllm.multimodal import MULTIMODAL_REGISTRY

23 +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange

24 from vllm.multimodal.utils import group_mm_inputs_by_modality

25 from vllm.sampling_params import SamplingType

26 from vllm.sequence import IntermediateTensors

@@ -43,7 +44,8 @@ from vllm.v1.utils import bind_kv_cache

44 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch

45 from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin

46 -from .utils import sanity_check_mm_encoder_outputs

47 +from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,

48 + scatter_mm_placeholders)

50 if TYPE_CHECKING:

51 import xgrammar as xgr

@@ -830,19 +832,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):

832 )

833 return metadata

834

833 - def _execute_encoder(self, scheduler_output: "SchedulerOutput"):

835 + def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):

836 scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs

837 if not scheduled_encoder_inputs:

838 return

839

840 # Batch the multi-modal inputs.

839 - mm_inputs: list[MultiModalKwargs] = []

840 - req_input_ids: list[tuple[str, int]] = []

841 + mm_inputs = list[MultiModalKwargs]()

842 + req_ids_pos = list[tuple[str, int, PlaceholderRange]]()

843 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():

844 req_state = self.requests[req_id]

843 - for input_id in encoder_input_ids:

844 - mm_inputs.append(req_state.mm_inputs[input_id])

845 - req_input_ids.append((req_id, input_id))

845 +

846 + for mm_input_id in encoder_input_ids:

847 + mm_inputs.append(req_state.mm_inputs[mm_input_id])

848 + req_ids_pos.append(

849 + (req_id, mm_input_id, req_state.mm_positions[mm_input_id]))

850

851 # Batch mm inputs as much as we can: if a request in the batch has

852 # multiple modalities or a different modality than the previous one,

@@ -878,16 +882,23 @@ class GPUModelRunner(LoRAModelRunnerMixin):

882 encoder_outputs.append(output)

883

884 # Cache the encoder outputs.

881 - for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):

885 + for (req_id, input_id, pos_info), output in zip(

886 + req_ids_pos,

887 + encoder_outputs,

888 + ):

889 if req_id not in self.encoder_cache:

890 self.encoder_cache[req_id] = {}

884 - self.encoder_cache[req_id][input_id] = output

891

886 - def _gather_encoder_outputs(

892 + self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(

893 + output,

894 + is_embed=pos_info.is_embed,

895 + )

896 +

897 + def _gather_mm_embeddings(

898 self,

899 scheduler_output: "SchedulerOutput",

900 ) -> list[torch.Tensor]:

890 - encoder_outputs: list[torch.Tensor] = []

901 + mm_embeds: list[torch.Tensor] = []

902 for req_id in self.input_batch.req_ids:

903 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[

904 req_id]

@@ -895,8 +906,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):

906 num_computed_tokens = req_state.num_computed_tokens

907 mm_positions = req_state.mm_positions

908 for i, pos_info in enumerate(mm_positions):

898 - start_pos = pos_info["offset"]

899 - num_encoder_tokens = pos_info["length"]

909 + start_pos = pos_info.offset

910 + num_encoder_tokens = pos_info.length

911

912 # The encoder output is needed if the two ranges overlap:

913 # [num_computed_tokens,

@@ -918,8 +929,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):

929 assert req_id in self.encoder_cache

930 assert i in self.encoder_cache[req_id]

931 encoder_output = self.encoder_cache[req_id][i]

921 - encoder_outputs.append(encoder_output[start_idx:end_idx])

922 - return encoder_outputs

932 +

933 + if (is_embed := pos_info.is_embed) is not None:

934 + is_embed = is_embed[start_idx:end_idx]

935 +

936 + mm_embeds_item = gather_mm_placeholders(

937 + encoder_output[start_idx:end_idx],

938 + is_embed=is_embed,

939 + )

940 + mm_embeds.append(mm_embeds_item)

941 + return mm_embeds

942

943 def get_model(self) -> nn.Module:

944 return self.model

@@ -984,10 +1003,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1003

1004 if self.is_multimodal_model:

1005 # Run the multimodal encoder if any.

987 - self._execute_encoder(scheduler_output)

988 - encoder_outputs = self._gather_encoder_outputs(scheduler_output)

1006 + self._execute_mm_encoder(scheduler_output)

1007 + mm_embeds = self._gather_mm_embeddings(scheduler_output)

1008 else:

990 - encoder_outputs = []

1009 + mm_embeds = []

1010

1011 # Prepare the decoder inputs.

1012 attn_metadata, logits_indices, spec_decode_metadata = (

@@ -1009,9 +1028,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):

1028 # embeddings), we always use embeddings (rather than token ids)

1029 # as input to the multimodal model, even when the input is text.

1030 input_ids = self.input_ids[:num_scheduled_tokens]

1012 - if encoder_outputs:

1031 + if mm_embeds:

1032 inputs_embeds = self.model.get_input_embeddings(

1014 - input_ids, encoder_outputs)

1033 + input_ids, mm_embeds)

1034 else:

1035 inputs_embeds = self.model.get_input_embeddings(input_ids)

1036 # TODO(woosuk): Avoid the copy. Optimize.

vllm/v1/worker/tpu_model_runner.py

+65 -21

@@ -19,7 +19,8 @@ from vllm.config import VllmConfig

19 from vllm.forward_context import set_forward_context

20 from vllm.logger import init_logger

21 from vllm.model_executor.model_loader import get_model

22 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

22 +from vllm.multimodal import MULTIMODAL_REGISTRY

23 +from vllm.multimodal.inputs import MultiModalKwargs, PlaceholderRange

24 from vllm.multimodal.utils import group_mm_inputs_by_modality

25 from vllm.sampling_params import SamplingType

26 from vllm.sequence import IntermediateTensors

@@ -36,7 +37,8 @@ from vllm.v1.sample.tpu.sampler import Sampler as TPUSampler

37 from vllm.v1.utils import bind_kv_cache

38 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch

39 -from .utils import sanity_check_mm_encoder_outputs

40 +from .utils import (gather_mm_placeholders, sanity_check_mm_encoder_outputs,

41 + scatter_mm_placeholders)

43 if TYPE_CHECKING:

44 from vllm.v1.core.sched.output import SchedulerOutput

@@ -507,19 +509,46 @@ class TPUModelRunner:

509 logits_indices = logits_indices.to(self.device)

510 return attn_metadata, logits_indices

511

510 - def _execute_encoder(self, scheduler_output: "SchedulerOutput"):

512 + def _scatter_placeholders(

513 + self,

514 + embeds: torch.Tensor,

515 + is_embed: Optional[torch.Tensor],

516 + ) -> torch.Tensor:

517 + if is_embed is None:

518 + return embeds

519 +

520 + placeholders = embeds.new_full(

521 + (is_embed.shape[0], embeds.shape[-1]),

522 + fill_value=torch.nan,

523 + )

524 + placeholders[is_embed] = embeds

525 + return placeholders

526 +

527 + def _gather_placeholders(

528 + self,

529 + placeholders: torch.Tensor,

530 + is_embed: Optional[torch.Tensor],

531 + ) -> torch.Tensor:

532 + if is_embed is None:

533 + return placeholders

534 +

535 + return placeholders[is_embed]

536 +

537 + def _execute_mm_encoder(self, scheduler_output: "SchedulerOutput"):

538 scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs

539 if not scheduled_encoder_inputs:

540 return

541

542 # Batch the multi-modal inputs.

516 - mm_inputs: list[MultiModalKwargs] = []

517 - req_input_ids: list[tuple[str, int]] = []

543 + mm_inputs = list[MultiModalKwargs]()

544 + req_ids_pos = list[tuple[str, int, PlaceholderRange]]()

545 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():

546 req_state = self.requests[req_id]

520 - for input_id in encoder_input_ids:

521 - mm_inputs.append(req_state.mm_inputs[input_id])

522 - req_input_ids.append((req_id, input_id))

547 +

548 + for mm_input_id in encoder_input_ids:

549 + mm_inputs.append(req_state.mm_inputs[mm_input_id])

550 + req_ids_pos.append(

551 + (req_id, mm_input_id, req_state.mm_positions[mm_input_id]))

552

553 # Batch mm inputs as much as we can: if a request in the batch has

554 # multiple modalities or a different modality than the previous one,

@@ -555,16 +584,23 @@ class TPUModelRunner:

584 encoder_outputs.append(output)

585

586 # Cache the encoder outputs.

558 - for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):

587 + for (req_id, input_id, pos_info), output in zip(

588 + req_ids_pos,

589 + encoder_outputs,

590 + ):

591 if req_id not in self.encoder_cache:

592 self.encoder_cache[req_id] = {}

561 - self.encoder_cache[req_id][input_id] = output

593

563 - def _gather_encoder_outputs(

594 + self.encoder_cache[req_id][input_id] = scatter_mm_placeholders(

595 + output,

596 + is_embed=pos_info.is_embed,

597 + )

598 +

599 + def _gather_mm_embeddings(

600 self,

601 scheduler_output: "SchedulerOutput",

602 ) -> list[torch.Tensor]:

567 - encoder_outputs: list[torch.Tensor] = []

603 + mm_embeds: list[torch.Tensor] = []

604 for req_id in self.input_batch.req_ids:

605 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[

606 req_id]

@@ -572,8 +608,8 @@ class TPUModelRunner:

608 num_computed_tokens = req_state.num_computed_tokens

609 mm_positions = req_state.mm_positions

610 for i, pos_info in enumerate(mm_positions):

575 - start_pos = pos_info["offset"]

576 - num_encoder_tokens = pos_info["length"]

611 + start_pos = pos_info.offset

612 + num_encoder_tokens = pos_info.length

613

614 # The encoder output is needed if the two ranges overlap:

615 # [num_computed_tokens,

@@ -595,8 +631,16 @@ class TPUModelRunner:

631 assert req_id in self.encoder_cache

632 assert i in self.encoder_cache[req_id]

633 encoder_output = self.encoder_cache[req_id][i]

598 - encoder_outputs.append(encoder_output[start_idx:end_idx])

599 - return encoder_outputs

634 +

635 + if (is_embed := pos_info.is_embed) is not None:

636 + is_embed = is_embed[start_idx:end_idx]

637 +

638 + mm_embeds_item = gather_mm_placeholders(

639 + encoder_output[start_idx:end_idx],

640 + is_embed=is_embed,

641 + )

642 + mm_embeds.append(mm_embeds_item)

643 + return mm_embeds

644

645 @torch.no_grad()

646 def execute_model(

@@ -612,10 +656,10 @@ class TPUModelRunner:

656

657 if self.is_multimodal_model:

658 # Run the multimodal encoder if any.

615 - self._execute_encoder(scheduler_output)

616 - encoder_outputs = self._gather_encoder_outputs(scheduler_output)

659 + self._execute_mm_encoder(scheduler_output)

660 + mm_embeds = self._gather_mm_embeddings(scheduler_output)

661 else:

618 - encoder_outputs = []

662 + mm_embeds = []

663

664 # Prepare inputs

665 attn_metadata, logits_indices = self._prepare_inputs(scheduler_output)

@@ -623,9 +667,9 @@ class TPUModelRunner:

667 # NOTE(woosuk): To unify token ids and soft tokens (vision

668 # embeddings), we always use embeddings (rather than token ids)

669 # as input to the multimodal model, even when the input is text.

626 - if encoder_outputs:

670 + if mm_embeds:

671 inputs_embeds = self.model.get_input_embeddings(

628 - self.input_ids, encoder_outputs)

672 + self.input_ids, mm_embeds)

673 else:

674 inputs_embeds = self.model.get_input_embeddings(self.input_ids)

675 input_ids = None

vllm/v1/worker/utils.py

+45

@@ -1,4 +1,6 @@

1 # SPDX-License-Identifier: Apache-2.0

2 +from typing import Optional

3 +

4 import torch

@@ -27,3 +29,46 @@ def sanity_check_mm_encoder_outputs(

29 f"but got tensors with shapes {[e.shape for e in mm_embeddings]} "

30 "instead. This is most likely due to incorrect implementation "

31 "of the model's `get_multimodal_embeddings` method.")

32 +

33 +

34 +def scatter_mm_placeholders(

35 + embeds: torch.Tensor,

36 + is_embed: Optional[torch.Tensor],

37 +) -> torch.Tensor:

38 + """

39 + Scatter the multimodal embeddings into a contiguous tensor that represents

40 + the placeholder tokens.

41 +

42 + :class:`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.

43 +

44 + Args:

45 + embeds: The multimodal embeddings.

46 + Shape: `(num_embeds, embed_dim)`

47 + is_embed: A boolean mask indicating which positions in the placeholder

48 + tokens need to be filled with multimodal embeddings.

49 + Shape: `(num_placeholders, num_embeds)`

50 + """

51 + if is_embed is None:

52 + return embeds

53 +

54 + placeholders = embeds.new_full(

55 + (is_embed.shape[0], embeds.shape[-1]),

56 + fill_value=torch.nan,

57 + )

58 + placeholders[is_embed] = embeds

59 + return placeholders

60 +

61 +

62 +def gather_mm_placeholders(

63 + placeholders: torch.Tensor,

64 + is_embed: Optional[torch.Tensor],

65 +) -> torch.Tensor:

66 + """

67 + Reconstructs the embeddings from the placeholder tokens.

68 +

69 + This is the operation of :func:`scatter_mm_placeholders`.

70 + """

71 + if is_embed is None:

72 + return placeholders

73 +

74 + return placeholders[is_embed]