@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 61f412187d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Bugfix] Re-enable Gemma3 for V1 (#14980)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 19, 2025 at 14:58 UTC 61f412187d972a006aef1653bfe348aeaefb6a0b

8 files changed +419 -175

docs/source/models/supported_models.md

+2 -5

@@ -768,7 +768,7 @@ See [this page](#generative-models) for more information on how to use generativ

768 * `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc.

769 * ✅︎

770 * ✅︎

771 - *

771 + * ⚠️

772 - * `GLM4VForCausalLM`<sup>^</sup>

773 * GLM-4V

774 * T + I

@@ -951,13 +951,10 @@ V0 correctly implements the model's attention pattern:

951

952 V1 currently uses a simplified attention pattern:

953 - Uses causal attention for all tokens, including image tokens

954 -- Generates reasonable outputs but does not match the original model's attention for text + image inputs

954 +- Generates reasonable outputs but does not match the original model's attention for text + image inputs, especially when `{"do_pan_and_scan": True}`

955 - Will be updated in the future to support the correct behavior

956 -- Does not support `"do_pan_and_scan": True`

956

957 This limitation exists because the model's mixed attention pattern (bidirectional for images, causal otherwise) is not yet supported by vLLM's attention backends.

959 -

960 -For these reasons, `Gemma3ForConditionalGeneration` is supported only on V0 at the moment.

958 :::

959

960 :::{note}

tests/multimodal/test_processing.py

+54 -1

@@ -19,7 +19,8 @@ from vllm.multimodal.processing import (PlaceholderFeaturesInfo,

19 apply_token_matches,

20 find_mm_placeholders,

21 find_text_matches, find_token_matches,

22 - iter_token_matches)

22 + iter_token_matches,

23 + replace_token_matches)

24 # yapf: enable

25 from vllm.multimodal.profiling import MultiModalProfiler

26 from vllm.transformers_utils.tokenizer import (AnyTokenizer,

@@ -89,6 +90,58 @@ def test_iter_token_matches(token_ids, match_ids, expected):

90 assert all(match_len == len(match_ids) for match_len in match_lens)

93 +# yapf: disable

94 +@pytest.mark.parametrize(

95 + ("token_ids", "match_ids", "new_ids", "expected"),

96 + [

97 + ([], [], [-1], []),

98 + ([], [32000], [-1], []),

99 + (

100 + [32000, 32000, 32000],

101 + [32000],

102 + [-1],

103 + [-1, -1, -1],

104 + ),

105 + (

106 + [32000, 32000, 32000],

107 + [32000, 32000],

108 + [-1],

109 + [-1, 32000],

110 + ),

111 + (

112 + [32000, 32000, 32000],

113 + [32000, 32000, 32000],

114 + [-1],

115 + [-1],

116 + ),

117 + (

118 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],

119 + [28747, 32000],

120 + [-1],

121 + [9833, -1, 32000, 32000, 9833, -1, 32000, 918],

122 + ),

123 + (

124 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],

125 + [28747, 32000, 32000, 32000],

126 + [-1],

127 + [9833, -1, 9833, 28747, 32000, 32000, 918],

128 + ),

129 + (

130 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],

131 + [28747, 0, 32000],

132 + [-1],

133 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],

134 + ),

135 + ],

136 +)

137 +# yapf: enable

138 +def test_replace_token_matches(token_ids, match_ids, new_ids, expected):

139 + result = replace_token_matches(token_ids, match_ids, new_ids)

140 +

141 + # Manually constructed results

142 + assert result == expected

143 +

144 +

145 # yapf: disable

146 @pytest.mark.parametrize(

147 ("prompt", "target_by_key", "expected_by_key"),

vllm/model_executor/models/gemma3_mm.py

+211 -36

@@ -1,34 +1,43 @@

1 # SPDX-License-Identifier: Apache-2.0

2 import math

3 -from typing import (Any, Iterable, Literal, Mapping, Optional, Sequence, Set,

4 - Tuple, TypedDict, Union)

3 +from collections.abc import Iterable, Mapping, Sequence

4 +from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union

6 import torch

7 from torch import nn

8 from transformers import BatchFeature, Gemma3Config, Gemma3Processor

9 from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs

11 +import vllm.envs as envs

12 from vllm.config import VllmConfig

13 from vllm.logger import init_logger

14 from vllm.model_executor.layers.layernorm import GemmaRMSNorm

15 from vllm.model_executor.layers.sampler import SamplerOutput

16 from vllm.model_executor.models.module_mapping import MultiModelKeys

17 from vllm.model_executor.sampling_metadata import SamplingMetadata

17 -from vllm.multimodal import MULTIMODAL_REGISTRY

18 -from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

18 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

19 +from vllm.multimodal.inputs import MultiModalFieldConfig

20 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

21 MultiModalDataItems)

22 +# yapf: disable

23 from vllm.multimodal.processing import (BaseMultiModalProcessor,

22 - BaseProcessingInfo, PromptReplacement,

23 - PromptUpdate, encode_tokens)

24 + BaseProcessingInfo, BoundPromptUpdate,

25 + PlaceholderFeaturesInfo,

26 + PromptReplacement, PromptTargetMatch,

27 + PromptUpdate, PromptUpdateDetails,

28 + encode_tokens, find_mm_placeholders,

29 + replace_token_matches)

30 +# yapf: enable

31 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

32 from vllm.sequence import IntermediateTensors

33 +from vllm.utils import flatten_2d_lists

35 from .interfaces import (MultiModalEmbeddings, SupportsLoRA,

28 - SupportsMultiModal, SupportsPP, SupportsV0Only)

36 + SupportsMultiModal, SupportsPP)

37 from .siglip import SiglipVisionModel

38 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

39 maybe_prefix, merge_multimodal_embeddings)

40 +from .vision import scatter_patch_features, select_patch_features

42 logger = init_logger(__name__)

@@ -37,13 +46,25 @@ class Gemma3ImagePixelInputs(TypedDict):

46 type: Literal["pixel_values"]

47 pixel_values: torch.Tensor

48 """

40 - Shape: `(num_crops_total, num_channels, height, width)`

49 + Shape: `(num_patches_total, num_channels, height, width)`

42 - `num_crops_total` is the total number of crops

51 + `num_patches_total` is the total number of patches

52 over each image over each prompt in the batch.

53 """

45 - num_crops: torch.Tensor

46 - """Shape: `(batch_size * num_images,)`"""

54 +

55 + num_patches: torch.Tensor

56 + """Shape: `(batch_size * num_images)`"""

57 +

58 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

59 + """

60 + A boolean mask indicating which image embeddings correspond

61 + to patch tokens.

62 +

63 + Shape: `(batch_size, num_images, num_embeds)`

64 + """

65 +

66 + num_embeds: Union[torch.Tensor, list[torch.Tensor]]

67 + """Shape: `(batch_size, num_images)`"""

70 Gemma3ImageInputs = Gemma3ImagePixelInputs

@@ -51,6 +72,9 @@ Gemma3ImageInputs = Gemma3ImagePixelInputs

73 class Gemma3ProcessingInfo(BaseProcessingInfo):

75 + def get_hf_config(self):

76 + return self.ctx.get_hf_config(Gemma3Config)

77 +

78 def get_hf_processor(self, **kwargs: object):

79 return self.ctx.get_hf_processor(Gemma3Processor, **kwargs)

@@ -114,6 +138,11 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

138 if not do_pan_and_scan:

139 return 0

140

141 + if envs.VLLM_USE_V1:

142 + logger.warning_once(

143 + "`do_pan_and_scan=True` has suboptimal results on V1 "

144 + "because of the simplified attention pattern being used.")

145 +

146 # Based on Gemma3ImageProcessor.pan_and_scan

147 if image_width >= image_height:

148 if image_width / image_height < pan_and_scan_min_ratio_to_activate:

@@ -154,7 +183,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

183 image_width: int,

184 image_height: int,

185 processor: Optional[Gemma3Processor],

157 - ) -> str:

186 + ) -> PromptUpdateDetails:

187 if processor is None:

188 processor = self.get_hf_processor()

189

@@ -175,7 +204,11 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

204 f"Here is the original image {image_token} and here are some "

205 f"crops to help you see better {crops_image_tokens}")

206

178 - return image_text.replace(image_token, processor.full_image_sequence)

207 + repl_full = image_text.replace(image_token,

208 + processor.full_image_sequence)

209 + repl_features = repl_full.strip("\n")

210 +

211 + return PromptUpdateDetails(full=repl_full, features=repl_features)

212

213 def get_num_image_tokens(

214 self,

@@ -193,7 +226,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):

226

227 image_repl_tokens = encode_tokens(

228 tokenizer,

196 - image_repl,

229 + image_repl.features,

230 add_special_tokens=False,

231 )

232 return len(image_repl_tokens)

@@ -240,12 +273,8 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):

273 num_images=num_images)

274 }

275

243 - # NOTE: We need to separate the image tokens here because

244 - # encode("\n\n\n\n") != encode("\n\n") * 2, which interferes

245 - # with the detection of prompt updates when the image tokens are

246 - # right next to each other

276 return ProcessorInputs(

248 - prompt_text=" ".join([image_token] * num_images),

277 + prompt_text=image_token * num_images,

278 mm_data=mm_data,

279 )

280

@@ -278,13 +307,39 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

307 ]

308 hf_processor = self.info.get_hf_processor(**mm_kwargs)

309

310 + image_repl_features = [

311 + self.info.get_image_repl(image_width=size.width,

312 + image_height=size.height,

313 + processor=hf_processor).features

314 + for size in image_sizes

315 + ]

316 +

317 + tokenizer = self.info.get_tokenizer()

318 + image_repls_feature_tokens = [

319 + tokenizer.encode(image_repl, add_special_tokens=False)

320 + for image_repl in image_repl_features

321 + ]

322 + num_embeds = [

323 + len(image_repl_feature_tokens)

324 + for image_repl_feature_tokens in image_repls_feature_tokens

325 + ]

326 + processed_outputs["num_embeds"] = torch.tensor(num_embeds)

327 +

328 + vocab = tokenizer.get_vocab()

329 + image_token_id = vocab[tokenizer.image_token]

330 +

331 + embed_is_patch = [

332 + torch.tensor(image_repl_tokens) == image_token_id

333 + for image_repl_tokens in image_repls_feature_tokens

334 + ]

335 + processed_outputs["embed_is_patch"] = embed_is_patch

336 +

337 num_crops = [

338 self.info.get_num_crops(image_width=size.width,

339 image_height=size.height,

340 processor=hf_processor)

341 for size in image_sizes

342 ]

287 -

343 processed_outputs["num_crops"] = torch.tensor(num_crops)

344

345 return processed_outputs

@@ -300,6 +355,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

355 pixel_values=MultiModalFieldConfig.flat_from_sizes(

356 "image", num_crops + 1),

357 num_crops=MultiModalFieldConfig.batched("image"),

358 + embed_is_patch=MultiModalFieldConfig.batched("image"),

359 + num_embeds=MultiModalFieldConfig.batched("image"),

360 )

361

362 def _get_prompt_updates(

@@ -329,6 +386,91 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):

386 )

387 ]

388

389 + def _apply_token_matches(

390 + self,

391 + prompt: list[int],

392 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

393 + mm_item_counts: Mapping[str, int],

394 + ) -> list[int]:

395 + token_ids = super()._apply_token_matches(

396 + prompt,

397 + mm_matches,

398 + mm_item_counts,

399 + )

400 +

401 + # "\n\n\n" and "\n\n\n\n" are single tokens

402 + # Since our replacement can insert "\n\n" next to "\n"

403 + # tokens, we have to combine them to be consistent with

404 + # the output of the tokenizer

405 + tokenizer = self.info.get_tokenizer()

406 + vocab = tokenizer.get_vocab()

407 + newline_1 = vocab["\n"]

408 + newline_2 = vocab["\n\n"]

409 + newline_3 = vocab["\n\n\n"]

410 + newline_4 = vocab["\n\n\n\n"]

411 +

412 + token_ids = replace_token_matches(

413 + token_ids,

414 + [newline_1, newline_2],

415 + [newline_3],

416 + )

417 + token_ids = replace_token_matches(

418 + token_ids,

419 + [newline_2, newline_1],

420 + [newline_3],

421 + )

422 + token_ids = replace_token_matches(

423 + token_ids,

424 + [newline_2, newline_2],

425 + [newline_4],

426 + )

427 +

428 + return token_ids

429 +

430 + def _find_mm_placeholders(

431 + self,

432 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],

433 + new_token_ids: list[int],

434 + mm_item_counts: Mapping[str, int],

435 + ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:

436 + # We need to detect "\n\n" inside "\n\n\n" and "\n\n\n\n"

437 + tokenizer = self.info.get_tokenizer()

438 + vocab = tokenizer.get_vocab()

439 + newline_1 = vocab["\n"]

440 + newline_2 = vocab["\n\n"]

441 + newline_3 = vocab["\n\n\n"]

442 + newline_4 = vocab["\n\n\n\n"]

443 +

444 + def get_repl_toks(tok: int) -> list[int]:

445 + if tok == newline_3:

446 + return [newline_1, newline_2]

447 + if tok == newline_4:

448 + return [newline_2, newline_2]

449 +

450 + return [tok]

451 +

452 + repl_token_ids = list[int]()

453 + repl_orig_idxs = list[int]()

454 + for orig_idx, orig_tok in enumerate(new_token_ids):

455 + repl_toks = get_repl_toks(orig_tok)

456 + repl_token_ids.extend(repl_toks)

457 + repl_orig_idxs.extend(orig_idx for _ in range(len(repl_toks)))

458 +

459 + repls = find_mm_placeholders(mm_prompt_updates, repl_token_ids,

460 + mm_item_counts)

461 +

462 + return {

463 + modality: [

464 + PlaceholderFeaturesInfo(

465 + modality=p.modality,

466 + item_idx=p.item_idx,

467 + start_idx=repl_orig_idxs[p.start_idx],

468 + tokens=p.tokens,

469 + ) for p in placeholders

470 + ]

471 + for modality, placeholders in repls.items()

472 + }

473 +

474

475 class Gemma3MultiModalProjector(nn.Module):

476

@@ -374,7 +516,7 @@ class Gemma3MultiModalProjector(nn.Module):

516 info=Gemma3ProcessingInfo,

517 dummy_inputs=Gemma3DummyInputsBuilder)

518 class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

377 - SupportsLoRA, SupportsV0Only):

519 + SupportsLoRA):

520 packed_modules_mapping = {

521 "qkv_proj": [

522 "q_proj",

@@ -415,6 +557,10 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

557 self.make_empty_intermediate_tensors = (

558 self.language_model.make_empty_intermediate_tensors)

559

560 + @property

561 + def dtype(self):

562 + return next(self.parameters()).dtype

563 +

564 @property

565 def sampler(self):

566 return self.language_model.sampler

@@ -438,6 +584,8 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

584 self, **kwargs: object) -> Optional[Gemma3ImageInputs]:

585 pixel_values = kwargs.pop("pixel_values", None)

586 num_crops = kwargs.pop("num_crops", None)

587 + embed_is_patch = kwargs.pop("embed_is_patch", None)

588 + num_embeds = kwargs.pop("num_embeds", None)

589 image_embeds = kwargs.pop("image_embeds", None)

590 assert image_embeds is None, "Gemma3 does not support image_embeds."

591 if pixel_values is None:

@@ -448,16 +596,26 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

596 f"Got type: {type(pixel_values)}")

597

598 if not isinstance(num_crops, (torch.Tensor, list)):

451 - raise ValueError("Incorrect type of num_crops values. "

599 + raise ValueError("Incorrect type of num_crops. "

600 f"Got type: {type(num_crops)}")

601

602 + if not isinstance(embed_is_patch, (torch.Tensor, list)):

603 + raise ValueError("Incorrect type of embed_is_patch. "

604 + f"Got type: {type(embed_is_patch)}")

605 +

606 + if not isinstance(num_embeds, (torch.Tensor, list)):

607 + raise ValueError("Incorrect type of num_embeds. "

608 + f"Got type: {type(num_embeds)}")

609 +

610 pixel_values = flatten_bn(pixel_values, concat=True)

611 num_crops = flatten_bn(num_crops, concat=True)

612

613 return Gemma3ImagePixelInputs(

614 type="pixel_values",

615 pixel_values=self._validate_pixel_values(pixel_values),

460 - num_crops=num_crops,

616 + num_patches=num_crops + 1,

617 + embed_is_patch=embed_is_patch,

618 + num_embeds=num_embeds,

619 )

620

621 def _image_pixels_to_features(

@@ -472,36 +630,51 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

630 def _process_image_input(

631 self,

632 image_input: Gemma3ImageInputs,

475 - ) -> torch.Tensor:

633 + ) -> tuple[torch.Tensor, ...]:

634 assert self.vision_tower is not None

635

636 pixel_values = image_input["pixel_values"]

479 - vision_outputs = self._image_pixels_to_features(

637 + num_patches = image_input["num_patches"]

638 +

639 + image_features = self._image_pixels_to_features(

640 self.vision_tower,

641 pixel_values,

642 )

483 - return self.multi_modal_projector(vision_outputs)

643 + image_embeds = self.multi_modal_projector(image_features)

644 +

645 + return image_embeds.split(num_patches.tolist())

646

647 def get_multimodal_embeddings(

648 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

649 image_input = self._parse_and_validate_image_input(**kwargs)

650 if image_input is None:

651 return None

490 - vision_embeddings = self._process_image_input(image_input)

491 - return vision_embeddings

652 +

653 + image_features = self._process_image_input(image_input)

654 +

655 + if kwargs.get("v0_path", False):

656 + return image_features

657 +

658 + return flatten_2d_lists(

659 + scatter_patch_features(*args) for args in zip(

660 + image_features,

661 + image_input["num_embeds"],

662 + image_input["embed_is_patch"],

663 + ))

664

665 def get_input_embeddings(

666 self,

667 input_ids: torch.Tensor,

668 multimodal_embeddings: Optional[MultiModalEmbeddings] = None,

669 ) -> torch.Tensor:

498 - if multimodal_embeddings is None:

499 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)

500 - else:

501 - inputs_embeds = self.language_model.get_input_embeddings(input_ids)

670 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)

671 + if multimodal_embeddings is not None:

672 inputs_embeds = merge_multimodal_embeddings(

503 - input_ids, inputs_embeds, multimodal_embeddings,

504 - self.config.image_token_index)

673 + input_ids,

674 + inputs_embeds,

675 + select_patch_features(multimodal_embeddings),

676 + self.config.image_token_index,

677 + )

678 return inputs_embeds

679

680 def forward(self,

@@ -516,6 +689,7 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

689 # NOTE: In v1, inputs_embeds is always generated at model runner, this

690 # condition is for v0 compatibility.

691 elif inputs_embeds is None:

692 + kwargs.update({"v0_path": True})

693 vision_embeddings = self.get_multimodal_embeddings(**kwargs)

694

695 inputs_embeds = self.get_input_embeddings(input_ids,

@@ -524,8 +698,9 @@ class Gemma3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

698 kwargs = self.prepare_attn_masks(

699 input_ids,

700 positions,

527 - mask_dtype=vision_embeddings.dtype,

528 - **kwargs)

701 + mask_dtype=self.dtype,

702 + **kwargs,

703 + )

704 input_ids = None

705

706 hidden_states = self.language_model.model(input_ids,

vllm/model_executor/models/llava.py

+18 -51

@@ -18,7 +18,7 @@ from transformers.models.pixtral import PixtralProcessor

19 from vllm.config import VllmConfig

20 from vllm.inputs import InputProcessingContext

21 -from vllm.jsontree import JSONTree, json_map_leaves

21 +from vllm.jsontree import json_map_leaves

22 from vllm.model_executor.layers.activation import get_act_fn

23 from vllm.model_executor.layers.linear import (ColumnParallelLinear,

24 RowParallelLinear)

@@ -27,8 +27,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler

27 from vllm.model_executor.sampling_metadata import SamplingMetadata

28 from vllm.multimodal import MULTIMODAL_REGISTRY

29 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

30 - MultiModalInputs, MultiModalKwargs,

31 - NestedTensors)

30 + MultiModalInputs, MultiModalKwargs)

31 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

32 ImageSize, MultiModalDataItems)

33 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -44,7 +43,8 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel

43 from .siglip import SiglipVisionModel

44 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

45 maybe_prefix, merge_multimodal_embeddings)

47 -from .vision import get_vision_encoder_info

46 +from .vision import (get_vision_encoder_info, scatter_patch_features,

47 + select_patch_features)

50 class LlavaImagePixelInputs(TypedDict):

@@ -76,7 +76,7 @@ class PixtralHFImagePixelInputs(TypedDict):

76 Shape: `(batch_size, num_images, num_embeds)`

77 """

79 - num_patches: Union[torch.Tensor, list[torch.Tensor]]

79 + num_embeds: Union[torch.Tensor, list[torch.Tensor]]

80 """Shape: `(batch_size, num_images)`"""

@@ -352,15 +352,15 @@ class PixtralHFMultiModalProcessor(

352 image_height=pixel_value.shape[-2],

353 ) for pixel_value in processed_outputs["pixel_values"]

354 ]

355 - num_patches = torch.tensor([(ncols + 1) * nrows

356 - for ncols, nrows in tile_sizes])

355 + num_embeds = torch.tensor([(ncols + 1) * nrows

356 + for ncols, nrows in tile_sizes])

357 # Each image may result to masks of different sizes, so we need to

358 - # later use `num_patches` to get per-image masks.

358 + # later use `num_embeds` to get per-image masks.

359 embed_is_patch = [

360 torch.tensor(([True] * ncols + [False]) * nrows)

361 for ncols, nrows in tile_sizes

362 ]

363 - processed_outputs["num_patches"] = num_patches

363 + processed_outputs["num_embeds"] = num_embeds

364 processed_outputs["embed_is_patch"] = embed_is_patch

365

366 return processed_outputs

@@ -372,7 +372,7 @@ class PixtralHFMultiModalProcessor(

372 ) -> Mapping[str, MultiModalFieldConfig]:

373 return dict(

374 pixel_values=MultiModalFieldConfig.batched("image"),

375 - num_patches=MultiModalFieldConfig.batched("image"),

375 + num_embeds=MultiModalFieldConfig.batched("image"),

376 embed_is_patch=MultiModalFieldConfig.batched("image"),

377 image_embeds=MultiModalFieldConfig.batched("image"),

378 )

@@ -621,16 +621,16 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

621 raise ValueError("Incorrect type of embed_is_patch. "

622 f"Got type: {type(embed_is_patch)}")

623

624 - num_patches = kwargs.pop("num_patches")

625 - if not isinstance(num_patches, (torch.Tensor, list)):

626 - raise ValueError("Incorrect type of num_patches. "

627 - f"Got type: {type(num_patches)}")

624 + num_embeds = kwargs.pop("num_embeds")

625 + if not isinstance(num_embeds, (torch.Tensor, list)):

626 + raise ValueError("Incorrect type of num_embeds. "

627 + f"Got type: {type(num_embeds)}")

628

629 return PixtralHFImagePixelInputs(

630 type="pixel_values_pixtral",

631 pixel_values=flatten_bn(pixel_values),

632 embed_is_patch=embed_is_patch,

633 - num_patches=num_patches,

633 + num_embeds=num_embeds,

634 )

635

636 return LlavaImagePixelInputs(

@@ -716,33 +716,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

716 image_embeds = torch.split(image_embeds, feature_sizes)

717 return image_embeds

718

719 - def _get_mm_embeds(

720 - self,

721 - features: torch.Tensor, # Shape: (num_patch, d)

722 - num_patches: torch.Tensor, # Shape: (num_images,)

723 - embed_is_patch: torch.Tensor, # Shape: (num_images, num_embeds)

724 - ) -> tuple[torch.Tensor, ...]:

725 - """Scatter the patch features into a contiguous tensor that corresponds

726 - to the embedding tokens defined by the multimodal processor.

727 -

728 - Mostly copied from `Molmo._get_mm_embeds`. See following fixme comment.

729 - """

730 - # Insert columns of nan values according to `embed_is_patch`. This work

731 - # ideally should be done in `_process_image_input`, but

732 - # `_process_image_input` is used in both V0 and V1 path. It's safer to

733 - # put the logic here.

734 - # FIXME: Move this logic to `_process_image_input` when v0 is

735 - # deprecated. Merge this function with `Molmo._get_mm_embeds`.

736 - num_patches_per_image: list[int] = num_patches.tolist()

737 -

738 - embeds_flat = features.new_full(

739 - (sum(num_patches_per_image), *features.shape[1:]),

740 - fill_value=torch.nan,

741 - )

742 - embeds_flat[embed_is_patch.view(-1)] = features

743 -

744 - return embeds_flat.split(num_patches_per_image)

745 -

719 def get_multimodal_embeddings(

720 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

721 image_input = self._parse_and_validate_image_input(**kwargs)

@@ -757,9 +730,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

730 return vision_embeddings

731

732 return flatten_2d_lists(

760 - self._get_mm_embeds(*args) for args in zip(

733 + scatter_patch_features(*args) for args in zip(

734 vision_embeddings,

762 - image_input["num_patches"],

735 + image_input["num_embeds"],

736 image_input["embed_is_patch"],

737 ))

738

@@ -770,16 +743,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

743 ) -> torch.Tensor:

744 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

745 if multimodal_embeddings is not None:

773 - # Extract the patch tokens

774 - patch_embeddings = json_map_leaves(

775 - lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),

776 - cast(JSONTree[torch.Tensor], multimodal_embeddings),

777 - )

778 -

746 inputs_embeds = merge_multimodal_embeddings(

747 input_ids,

748 inputs_embeds,

782 - cast(NestedTensors, patch_embeddings),

749 + select_patch_features(multimodal_embeddings),

750 self.config.image_token_index,

751 )

752 return inputs_embeds

vllm/model_executor/models/molmo.py

+4 -11

@@ -4,7 +4,7 @@ import math

4 from collections.abc import Iterable, Mapping, Sequence

5 from dataclasses import dataclass

6 from functools import cached_property, partial

7 -from typing import List, Optional, Set, Tuple, TypedDict, Union, cast

7 +from typing import List, Optional, Set, Tuple, TypedDict, Union

9 import numpy as np

10 import torch

@@ -24,7 +24,6 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,

24 get_tensor_model_parallel_world_size,

25 split_tensor_along_last_dim,

26 tensor_model_parallel_all_gather)

27 -from vllm.jsontree import JSONTree, json_map_leaves

27 from vllm.model_executor import SamplingMetadata

28 from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,

29 SiluAndMul)

@@ -42,8 +41,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (

41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

42 from vllm.model_executor.models.module_mapping import MultiModelKeys

43 from vllm.multimodal import MULTIMODAL_REGISTRY

45 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

46 - NestedTensors)

44 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

45 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

46 MultiModalDataItems)

47 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -59,6 +57,7 @@ from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,

57 is_pp_missing_parameter,

58 make_empty_intermediate_tensors_factory, make_layers,

59 maybe_prefix, merge_multimodal_embeddings)

60 +from .vision import select_patch_features

62 # TODO: hard-coded for now. Consider making it configurable.

63 VIT_LAYERS = [-2, -9]

@@ -1602,16 +1601,10 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1601 if multimodal_embeddings is not None:

1602 assert self.img_patch_id is not None

1603

1605 - # Extract the patch tokens scattered in _get_mm_embeds

1606 - patch_embeddings = json_map_leaves(

1607 - lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),

1608 - cast(JSONTree[torch.Tensor], multimodal_embeddings),

1609 - )

1610 -

1604 inputs_embeds = merge_multimodal_embeddings(

1605 input_ids,

1606 inputs_embeds,

1614 - cast(NestedTensors, patch_embeddings),

1607 + select_patch_features(multimodal_embeddings),

1608 self.img_patch_id,

1609 )

1610 return inputs_embeds

vllm/model_executor/models/pixtral.py

+16 -48

@@ -4,7 +4,7 @@ import math

4 from collections.abc import Iterable, Mapping, Sequence

5 from dataclasses import dataclass, fields

6 from functools import cached_property

7 -from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union, cast

7 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union

9 import torch

10 import torch.nn as nn

@@ -22,7 +22,6 @@ from transformers.tokenization_utils_base import TextInput

23 from vllm.config import VllmConfig

24 from vllm.distributed import divide, get_tensor_model_parallel_world_size

25 -from vllm.jsontree import JSONTree, json_map_leaves

25 from vllm.model_executor.layers.activation import get_act_and_mul_fn

26 from vllm.model_executor.layers.layernorm import RMSNorm

27 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,

@@ -48,7 +47,8 @@ from vllm.utils import flatten_2d_lists

47 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

48 from .utils import (flatten_bn, init_vllm_registered_model, maybe_prefix,

49 merge_multimodal_embeddings)

51 -from .vision import VisionEncoderInfo, resolve_visual_encoder_outputs

50 +from .vision import (VisionEncoderInfo, resolve_visual_encoder_outputs,

51 + scatter_patch_features, select_patch_features)

53 try:

54 from xformers import ops as xops

@@ -77,7 +77,7 @@ class PixtralImagePixelInputs(TypedDict):

77 Shape: `(batch_size, num_images, num_embeds)`

78 """

80 - num_patches: Union[torch.Tensor, list[torch.Tensor]]

80 + num_embeds: Union[torch.Tensor, list[torch.Tensor]]

81 """Shape: `(batch_size, num_images)`"""

@@ -153,7 +153,7 @@ class PixtralProcessorAdapter:

153 images_processed = list[torch.Tensor]()

154 images_tokens = list[torch.Tensor]()

155 images_embed_is_patch = list[torch.Tensor]()

156 - images_num_patches = list[int]()

156 + images_num_embeds = list[int]()

157

158 for image in images:

159 image_inputs = self.image_processor(ImageChunk(image=image))

@@ -163,13 +163,13 @@ class PixtralProcessorAdapter:

163 images_processed.append(image_processed)

164 images_tokens.append(image_tokens)

165 images_embed_is_patch.append(image_tokens == image_token_id)

166 - images_num_patches.append(len(image_tokens))

166 + images_num_embeds.append(len(image_tokens))

167

168 return {

169 "input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),

170 "images": images_processed,

171 "embed_is_patch": images_embed_is_patch,

172 - "num_patches": torch.tensor(images_num_patches),

172 + "num_embeds": torch.tensor(images_num_embeds),

173 }

174

175

@@ -273,7 +273,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]

273 return dict(

274 images=MultiModalFieldConfig.batched("image"),

275 embed_is_patch=MultiModalFieldConfig.batched("image"),

276 - num_patches=MultiModalFieldConfig.batched("image"),

276 + num_embeds=MultiModalFieldConfig.batched("image"),

277 )

278

279 def _get_prompt_updates(

@@ -394,16 +394,16 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

394 raise ValueError("Incorrect type of embed_is_patch. "

395 f"Got type: {type(embed_is_patch)}")

396

397 - num_patches = kwargs.pop("num_patches")

398 - if not isinstance(num_patches, (torch.Tensor, list)):

399 - raise ValueError("Incorrect type of num_patches. "

400 - f"Got type: {type(num_patches)}")

397 + num_embeds = kwargs.pop("num_embeds")

398 + if not isinstance(num_embeds, (torch.Tensor, list)):

399 + raise ValueError("Incorrect type of num_embeds. "

400 + f"Got type: {type(num_embeds)}")

401

402 return PixtralImagePixelInputs(

403 type="pixel_values",

404 images=flatten_bn(images),

405 embed_is_patch=embed_is_patch,

406 - num_patches=num_patches,

406 + num_embeds=num_embeds,

407 )

408

409 def _process_image_input(

@@ -433,33 +433,6 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

433 image_embeds = torch.split(image_embeds, feature_sizes)

434 return image_embeds

435

436 - def _get_mm_embeds(

437 - self,

438 - features: torch.Tensor, # Shape: (num_patch, d)

439 - num_patches: torch.Tensor, # Shape: (num_images,)

440 - embed_is_patch: torch.Tensor, # Shape: (num_images, num_embeds)

441 - ) -> tuple[torch.Tensor, ...]:

442 - """Scatter the patch features into a contiguous tensor that corresponds

443 - to the embedding tokens defined by the multimodal processor.

444 -

445 - Mostly copied from `Molmo._get_mm_embeds`. See following fixme comment.

446 - """

447 - # Insert columns of nan values according to `embed_is_patch`. This work

448 - # ideally should be done in `_process_image_input`, but

449 - # `_process_image_input` is used in both V0 and V1 path. It's safer to

450 - # put the logic here.

451 - # FIXME: Move this logic to `_process_image_input` when v0 is

452 - # deprecated. Merge this function with `Molmo._get_mm_embeds`.

453 - num_patches_per_image: list[int] = num_patches.tolist()

454 -

455 - embeds_flat = features.new_full(

456 - (sum(num_patches_per_image), *features.shape[1:]),

457 - fill_value=torch.nan,

458 - )

459 - embeds_flat[embed_is_patch.view(-1)] = features

460 -

461 - return embeds_flat.split(num_patches_per_image)

462 -

436 def get_multimodal_embeddings(

437 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

438 image_input = self._parse_and_validate_image_input(**kwargs)

@@ -472,9 +445,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

445 return image_features

446

447 return flatten_2d_lists(

475 - self._get_mm_embeds(*args) for args in zip(

448 + scatter_patch_features(*args) for args in zip(

449 image_features,

477 - image_input["num_patches"],

450 + image_input["num_embeds"],

451 image_input["embed_is_patch"],

452 ))

453

@@ -485,15 +458,10 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,

458 ) -> torch.Tensor:

459 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

460 if multimodal_embeddings is not None:

488 - # Extract the patch tokens

489 - patch_embeddings = json_map_leaves(

490 - lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),

491 - cast(JSONTree[torch.Tensor], multimodal_embeddings),

492 - )

461 inputs_embeds = merge_multimodal_embeddings(

462 input_ids,

463 inputs_embeds,

496 - cast(NestedTensors, patch_embeddings),

464 + select_patch_features(multimodal_embeddings),

465 self.vision_args.image_token_id,

466 )

467 return inputs_embeds

vllm/model_executor/models/vision.py

+49 -1

@@ -1,7 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

3 from abc import ABC, abstractmethod

4 -from typing import Final, Generic, Optional, Protocol, TypeVar, Union

4 +from typing import Final, Generic, Optional, Protocol, TypeVar, Union, cast

6 import torch

7 from transformers import PretrainedConfig

@@ -9,9 +9,12 @@ from transformers import PretrainedConfig

9 import vllm.envs as envs

10 from vllm.attention.selector import (backend_name_to_enum,

11 get_global_forced_attn_backend)

12 +from vllm.jsontree import JSONTree, json_map_leaves

13 from vllm.logger import init_logger

14 from vllm.platforms import _Backend, current_platform

16 +from .interfaces import MultiModalEmbeddings

17 +

18 logger = init_logger(__name__)

20 _C = TypeVar("_C", bound=PretrainedConfig)

@@ -148,3 +151,48 @@ def resolve_visual_encoder_outputs(

151 if post_layer_norm is not None and uses_last_layer:

152 hs_pool[-1] = post_layer_norm(encoder_outputs)

153 return torch.cat(hs_pool, dim=-1)

154 +

155 +

156 +def scatter_patch_features(

157 + features: torch.Tensor,

158 + num_embeds: torch.Tensor,

159 + embed_is_patch: torch.Tensor,

160 +) -> tuple[torch.Tensor, ...]:

161 + """

162 + Scatter the patch features into a contiguous tensor that corresponds

163 + to the embedding tokens defined by the multimodal processor.

164 +

165 + The rest of the values in the tensor are set to NaN so that they

166 + can be filtered out by :func`select_patch_features`.

167 +

168 + Args:

169 + features: The patch features, concatenated across each image.

170 + Shape: `(num_patch, feature_depth)`

171 + num_embeds: The number of image embeddings for each image.

172 + Shape: `(num_images,)`

173 + embed_is_patch: A boolean mask indicating which image embeddings

174 + correspond to patch tokens for each image.

175 + Shape: `(num_images, num_embeds)`

176 + """

177 + num_embeds_per_image: list[int] = num_embeds.tolist()

178 +

179 + embeds_flat = features.new_full(

180 + (sum(num_embeds_per_image), features.shape[-1]),

181 + fill_value=torch.nan,

182 + )

183 + embeds_flat[embed_is_patch.view(-1)] = features.flatten(0, -2)

184 +

185 + return embeds_flat.split(num_embeds_per_image)

186 +

187 +

188 +def select_patch_features(

189 + multimodal_embeddings: MultiModalEmbeddings) -> MultiModalEmbeddings:

190 + """

191 + Given the outputs of :func:`scatter_patch_features`, return only

192 + the values that correspond to patch features.

193 + """

194 + selected_features = json_map_leaves(

195 + lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),

196 + cast(JSONTree[torch.Tensor], multimodal_embeddings),

197 + )

198 + return cast(MultiModalEmbeddings, selected_features)

vllm/multimodal/processing.py

+65 -22

@@ -511,8 +511,35 @@ def iter_token_matches(

511 start_idx += 1

512

513

514 +def replace_token_matches(

515 + token_ids: list[int],

516 + match_ids: list[int],

517 + new_ids: list[int],

518 +) -> list[int]:

519 + """

520 + Replace each occurrence of :code:`match_ids` in :code:`token_ids`

521 + with :code:`new_ids`.

522 +

523 + Note that empty matches are ignored.

524 + """

525 + out_seqs = list[list[int]]()

526 + prev_end_idx = 0

527 +

528 + for match in iter_token_matches(token_ids, match_ids):

529 + start_idx = match.start_idx

530 + end_idx = match.end_idx

531 +

532 + out_seqs.append(token_ids[prev_end_idx:start_idx])

533 + out_seqs.append(new_ids)

534 + prev_end_idx = end_idx

535 +

536 + out_seqs.append(token_ids[prev_end_idx:])

537 +

538 + return flatten_2d_lists(out_seqs)

539 +

540 +

541 @dataclass(repr=False)

515 -class _PromptTargetMatch(ABC):

542 +class PromptTargetMatch(ABC):

543 _origin: BoundPromptUpdate

544

545 @property

@@ -535,7 +562,7 @@ class _PromptTargetMatch(ABC):

562

563

564 @dataclass(repr=False)

538 -class _PromptTargetIndexMatch(_PromptTargetMatch):

565 +class _PromptTargetIndexMatch(PromptTargetMatch):

566 match_idx: int

567

568 @property

@@ -548,7 +575,7 @@ class _PromptTargetIndexMatch(_PromptTargetMatch):

575

576

577 @dataclass(repr=False)

551 -class _PromptTargetTokenMatch(_PromptTargetMatch):

578 +class _PromptTargetTokenMatch(PromptTargetMatch):

579 match: _TokenMatch

580

581 @property

@@ -561,7 +588,7 @@ class _PromptTargetTokenMatch(_PromptTargetMatch):

588

589

590 @dataclass(repr=False)

564 -class _PromptTargetTextMatch(_PromptTargetMatch):

591 +class _PromptTargetTextMatch(PromptTargetMatch):

592 match: re.Match[str]

593

594 @property

@@ -594,7 +621,7 @@ class PlaceholderFeaturesInfo:

621 def find_token_matches(

622 prompt: list[int],

623 prompt_updates: Sequence[BoundPromptUpdate],

597 -) -> Sequence[_PromptTargetMatch]:

624 +) -> Sequence[PromptTargetMatch]:

625 """Return each target of :code:`prompt_updates` found in :code:`prompt`."""

626

627 def get_matches(update: BoundPromptUpdate):

@@ -620,7 +647,7 @@ def find_token_matches(

647 def find_text_matches(

648 prompt: str,

649 prompt_updates: Sequence[BoundPromptUpdate],

623 -) -> Sequence[_PromptTargetMatch]:

650 +) -> Sequence[PromptTargetMatch]:

651 """Return each target of :code:`prompt_updates` found in :code:`prompt`."""

652

653 def get_matches(update: BoundPromptUpdate):

@@ -645,15 +672,15 @@ def find_text_matches(

672

673 def _resolve_matches(

674 prompt: PromptSeq,

648 - mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],

649 -) -> list[_PromptTargetMatch]:

675 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

676 +) -> list[PromptTargetMatch]:

677 """

678 Resolve :code:`mm_matches` to ensure that there are no overlapping matches,

679 and sort them such that earlier matches take priority over later ones.

680 """

681 matches = [m for matches in mm_matches.values() for m in matches]

682

656 - seen_matches: list[Optional[_PromptTargetMatch]] = [None] * len(prompt)

683 + seen_matches: list[Optional[PromptTargetMatch]] = [None] * len(prompt)

684

685 for match in matches:

686 for idx in range(match.start_idx, match.end_idx):

@@ -669,7 +696,7 @@ def _resolve_matches(

696

697 def _apply_matches(

698 prompt: _S,

672 - mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],

699 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

700 mm_item_counts: Mapping[str, int],

701 ) -> list[_S]:

702 """Apply the updates in :code:`mm_matches` to :code:`prompt`."""

@@ -718,7 +745,7 @@ def _apply_matches(

745

746 def apply_token_matches(

747 prompt: list[int],

721 - mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],

748 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

749 mm_item_counts: Mapping[str, int],

750 ) -> list[int]:

751 """Apply the updates in :code:`mm_matches` to :code:`prompt`."""

@@ -732,7 +759,7 @@ def apply_token_matches(

759

760 def apply_text_matches(

761 prompt: str,

735 - mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],

762 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

763 mm_item_counts: Mapping[str, int],

764 ) -> str:

765 """Apply the updates in :code:`mm_matches` to :code:`prompt`."""

@@ -1055,14 +1082,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1082 Given the original multi-modal items for this modality

1083 and HF-processed data, output the updates to perform.

1084

1058 - Notes:

1059 - - You should not assume that HF processor always performs prompt

1060 - updates: in :meth:`_apply_hf_processor_missing`, this method

1061 - is called on text-only and multimodal-only inputs separately,

1062 - instead of passing them in the same call.

1063 - - The update information returned by this method is also used to

1064 - determine the placeholder token positions for each multi-modal

1065 - item.

1085 + The information returned by this method is used to update token inputs

1086 + which bypass the HF processor. It is also used to update the output of

1087 + HF processor if the HF process does not apply prompt updates to text

1088 + inputs.

1089 +

1090 + Moreover, this information is critical to determine the token positions

1091 + in order to construct :class:`~vllm-multimodal.input.PlaceholderRange`

1092 + for each multi-modal item.

1093 """

1094 raise NotImplementedError

1095

@@ -1357,6 +1384,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1384 it = (update.bind(tokenizer) for update in prompt_updates)

1385 return dict(full_groupby_modality(it))

1386

1387 + def _apply_token_matches(

1388 + self,

1389 + prompt: list[int],

1390 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

1391 + mm_item_counts: Mapping[str, int],

1392 + ) -> list[int]:

1393 + return apply_token_matches(prompt, mm_matches, mm_item_counts)

1394 +

1395 + def _apply_text_matches(

1396 + self,

1397 + prompt: str,

1398 + mm_matches: Mapping[str, Sequence[PromptTargetMatch]],

1399 + mm_item_counts: Mapping[str, int],

1400 + ) -> str:

1401 + return apply_text_matches(prompt, mm_matches, mm_item_counts)

1402 +

1403 def _apply_prompt_updates(

1404 self,

1405 token_ids: list[int],

@@ -1388,7 +1431,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1431 mm_match_counts.get(modality, 0) >= item_count

1432 for modality, item_count in mm_item_counts.items()

1433 ): # yapf: disable

1391 - token_ids = apply_token_matches(

1434 + token_ids = self._apply_token_matches(

1435 token_ids,

1436 mm_token_matches,

1437 mm_item_counts,

@@ -1406,7 +1449,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1449 modality: find_text_matches(text, updates)

1450 for modality, updates in mm_prompt_updates.items()

1451 }

1409 - text = apply_text_matches(

1452 + text = self._apply_text_matches(

1453 text,

1454 mm_text_matches,

1455 mm_item_counts,