@tduncan / Nvidia-Optimization-Stack-VLLM / commits / cb09dd7488

[Core][Multimodal] Skip redundant placeholder scan when token match succeeds (#52925)

Co-authored-by: shenyiqin <shenyiqin1@huawei.com>

Yiqin committed Aug 20, 2026 at 23:04 UTC cb09dd7488d2ad13dbf610aa2a580af2e7b050b5
4 files changed +539 -175
tests/multimodal/test_processing.py
+356 -150
@@ -17,6 +17,7 @@ from vllm.multimodal.processing.processor import (
17 PromptInsertion,
18 PromptReplacement,
19 _apply_matches,
20 + _apply_token_matches_with_placeholders,
21 apply_text_matches,
22 apply_token_matches,
23 find_mm_placeholders,
@@ -567,171 +568,309 @@ def test_find_update_text(
568 assert new_prompt == expected
569
570
570 -@pytest.mark.parametrize(
571 - ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
572 - [
573 - # Tokenized test cases of `test_find_update_text`
574 - # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
575 - (
576 - [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
577 - {
578 - # We use `<image>` before `Image:` to test matches that
579 - # occur out of order
580 - "pattern_1": [32000],
581 - "pattern_2": [9833, 28747],
582 - "pattern_3": [918],
571 +FIND_UPDATE_TOKENS_TEST_CASES = [
572 + # Tokenized test cases of `test_find_update_text`
573 + # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
574 + (
575 + [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
576 + {
577 + # We use `<image>` before `Image:` to test matches that
578 + # occur out of order
579 + "pattern_1": [32000],
580 + "pattern_2": [9833, 28747],
581 + "pattern_3": [918],
582 + },
583 + {
584 + # Test whether target is confused with replacement
585 + "pattern_1": [32000, 32000],
586 + # Test empty replacement
587 + "pattern_2": [],
588 + # Test dynamic replacement (beyond the form of `unit * count`)
589 + "pattern_3": [1550, 918, 1550],
590 + },
591 + {
592 + PromptInsertion: {
593 + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
594 + 1: [
595 + 1,
596 + 9833,
597 + 28747,
598 + 32000,
599 + 32000,
600 + 32000,
601 + 9833,
602 + 28747,
603 + 32000,
604 + 32000,
605 + 918,
606 + 1550,
607 + 918,
608 + 1550,
609 + ], # noqa: E501
610 + 2: [
611 + 1,
612 + 9833,
613 + 28747,
614 + 32000,
615 + 32000,
616 + 32000,
617 + 32000,
618 + 32000,
619 + 9833,
620 + 28747,
621 + 32000,
622 + 32000,
623 + 918,
624 + 1550,
625 + 918,
626 + 1550,
627 + 1550,
628 + 918,
629 + 1550,
630 + ], # noqa: E501
631 },
584 - {
585 - # Test whether target is confused with replacement
586 - "pattern_1": [32000, 32000],
587 - # Test empty replacement
588 - "pattern_2": [],
589 - # Test dynamic replacement (beyond the form of `unit * count`)
590 - "pattern_3": [1550, 918, 1550],
632 + PromptReplacement: {
633 + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
634 + 1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
635 + 2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
636 },
592 - {
593 - PromptInsertion: {
594 - 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
595 - 1: [
596 - 1,
597 - 9833,
598 - 28747,
599 - 32000,
600 - 32000,
601 - 32000,
602 - 9833,
603 - 28747,
604 - 32000,
605 - 32000,
606 - 918,
607 - 1550,
608 - 918,
609 - 1550,
610 - ], # noqa: E501
611 - 2: [
612 - 1,
613 - 9833,
614 - 28747,
615 - 32000,
616 - 32000,
617 - 32000,
618 - 32000,
619 - 32000,
620 - 9833,
621 - 28747,
622 - 32000,
623 - 32000,
624 - 918,
625 - 1550,
626 - 918,
627 - 1550,
628 - 1550,
629 - 918,
630 - 1550,
631 - ], # noqa: E501
632 - },
633 - PromptReplacement: {
634 - 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
635 - 1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
636 - 2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
637 - },
637 + },
638 + ),
639 + # Test index targets
640 + (
641 + [],
642 + {
643 + "pattern_1": PromptIndexTargets.start(),
644 + "pattern_2": PromptIndexTargets.prefix([32000]),
645 + "pattern_3": PromptIndexTargets.end(),
646 + },
647 + {
648 + "pattern_1": [-1],
649 + "pattern_2": [-2],
650 + "pattern_3": [-3],
651 + },
652 + {
653 + PromptInsertion: {
654 + 0: [],
655 + 1: [-1, -3],
656 + 2: [-1, -1, -3, -3],
657 },
639 - ),
640 - # Test index targets
641 - (
642 - [],
643 - {
644 - "pattern_1": PromptIndexTargets.start(),
645 - "pattern_2": PromptIndexTargets.prefix([32000]),
646 - "pattern_3": PromptIndexTargets.end(),
658 + PromptReplacement: {
659 + 0: [],
660 + 1: [-1, -3],
661 + 2: [-1, -1, -3, -3],
662 },
648 - {
649 - "pattern_1": [-1],
650 - "pattern_2": [-2],
651 - "pattern_3": [-3],
663 + },
664 + ),
665 + (
666 + [32000],
667 + {
668 + "pattern_1": PromptIndexTargets.start(),
669 + "pattern_2": PromptIndexTargets.prefix([32000]),
670 + "pattern_3": PromptIndexTargets.end(),
671 + },
672 + {
673 + "pattern_1": [-1],
674 + "pattern_2": [-2],
675 + "pattern_3": [-3],
676 + },
677 + {
678 + PromptInsertion: {
679 + 0: [32000],
680 + 1: [-1, 32000, -2, -3],
681 + 2: [-1, -1, 32000, -2, -2, -3, -3],
682 },
653 - {
654 - PromptInsertion: {
655 - 0: [],
656 - 1: [-1, -3],
657 - 2: [-1, -1, -3, -3],
658 - },
659 - PromptReplacement: {
660 - 0: [],
661 - 1: [-1, -3],
662 - 2: [-1, -1, -3, -3],
663 - },
683 + PromptReplacement: {
684 + 0: [32000],
685 + 1: [-1, 32000, -2, -3],
686 + 2: [-1, -1, 32000, -2, -2, -3, -3],
687 },
665 - ),
666 - (
667 - [32000],
668 - {
669 - "pattern_1": PromptIndexTargets.start(),
670 - "pattern_2": PromptIndexTargets.prefix([32000]),
671 - "pattern_3": PromptIndexTargets.end(),
688 + },
689 + ),
690 + # Test different replacement per item
691 + (
692 + [32000, 32000, 32000],
693 + {
694 + "pattern_1": [32000],
695 + },
696 + {
697 + "pattern_1": lambda idx: [-(idx + 1)],
698 + },
699 + {
700 + PromptInsertion: {
701 + 0: [32000, 32000, 32000],
702 + 1: [32000, -1, 32000, 32000],
703 + 2: [32000, -1, -2, 32000, 32000],
704 },
673 - {
674 - "pattern_1": [-1],
675 - "pattern_2": [-2],
676 - "pattern_3": [-3],
705 + PromptReplacement: {
706 + 0: [32000, 32000, 32000],
707 + 1: [-1, 32000, 32000],
708 + 2: [-1, -2, 32000],
709 },
678 - {
679 - PromptInsertion: {
680 - 0: [32000],
681 - 1: [-1, 32000, -2, -3],
682 - 2: [-1, -1, 32000, -2, -2, -3, -3],
683 - },
684 - PromptReplacement: {
685 - 0: [32000],
686 - 1: [-1, 32000, -2, -3],
687 - 2: [-1, -1, 32000, -2, -2, -3, -3],
688 - },
710 + },
711 + ),
712 + (
713 + [32000, 32000, 32000],
714 + {
715 + "pattern_1": PromptIndexTargets.prefix([32000]),
716 + },
717 + {
718 + "pattern_1": lambda idx: [-(idx + 1)],
719 + },
720 + {
721 + PromptInsertion: {
722 + 0: [32000, 32000, 32000],
723 + 1: [32000, -1, 32000, 32000],
724 + 2: [32000, -1, -2, 32000, 32000],
725 },
690 - ),
691 - # Test different replacement per item
692 - (
693 - [32000, 32000, 32000],
694 - {
695 - "pattern_1": [32000],
726 + PromptReplacement: {
727 + 0: [32000, 32000, 32000],
728 + 1: [32000, -1, 32000, 32000],
729 + 2: [32000, -1, -2, 32000, 32000],
730 },
697 - {
698 - "pattern_1": lambda idx: [-(idx + 1)],
731 + },
732 + ),
733 +]
734 +
735 +
736 +def _placeholder(modality, item_idx, start_idx, tokens):
737 + return PlaceholderFeaturesInfo(
738 + modality=modality,
739 + item_idx=item_idx,
740 + start_idx=start_idx,
741 + tokens=tokens,
742 + is_embed=None,
743 + )
744 +
745 +
746 +FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED = [
747 + {
748 + PromptInsertion: {
749 + 0: {},
750 + 1: {
751 + "pattern_1": [_placeholder("pattern_1", 0, 4, [32000, 32000])],
752 + "pattern_3": [_placeholder("pattern_3", 0, 11, [1550, 918, 1550])],
753 },
700 - {
701 - PromptInsertion: {
702 - 0: [32000, 32000, 32000],
703 - 1: [32000, -1, 32000, 32000],
704 - 2: [32000, -1, -2, 32000, 32000],
705 - },
706 - PromptReplacement: {
707 - 0: [32000, 32000, 32000],
708 - 1: [-1, 32000, 32000],
709 - 2: [-1, -2, 32000],
710 - },
754 + 2: {
755 + "pattern_1": [
756 + _placeholder("pattern_1", 0, 4, [32000, 32000]),
757 + _placeholder("pattern_1", 1, 6, [32000, 32000]),
758 + ],
759 + "pattern_3": [
760 + _placeholder("pattern_3", 0, 13, [1550, 918, 1550]),
761 + _placeholder("pattern_3", 1, 16, [1550, 918, 1550]),
762 + ],
763 },
712 - ),
713 - (
714 - [32000, 32000, 32000],
715 - {
716 - "pattern_1": PromptIndexTargets.prefix([32000]),
764 + },
765 + PromptReplacement: {
766 + 0: {},
767 + 1: {
768 + "pattern_1": [_placeholder("pattern_1", 0, 1, [32000, 32000])],
769 + "pattern_3": [_placeholder("pattern_3", 0, 7, [1550, 918, 1550])],
770 },
718 - {
719 - "pattern_1": lambda idx: [-(idx + 1)],
771 + 2: {},
772 + },
773 + },
774 + {
775 + PromptInsertion: {0: {}, 1: {}, 2: {}},
776 + PromptReplacement: {0: {}, 1: {}, 2: {}},
777 + },
778 + {
779 + PromptInsertion: {
780 + 0: {},
781 + 1: {
782 + "pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
783 + "pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
784 + "pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
785 },
721 - {
722 - PromptInsertion: {
723 - 0: [32000, 32000, 32000],
724 - 1: [32000, -1, 32000, 32000],
725 - 2: [32000, -1, -2, 32000, 32000],
726 - },
727 - PromptReplacement: {
728 - 0: [32000, 32000, 32000],
729 - 1: [32000, -1, 32000, 32000],
730 - 2: [32000, -1, -2, 32000, 32000],
731 - },
786 + 2: {
787 + "pattern_1": [
788 + _placeholder("pattern_1", 0, 0, [-1]),
789 + _placeholder("pattern_1", 1, 1, [-1]),
790 + ],
791 + "pattern_2": [
792 + _placeholder("pattern_2", 0, 3, [-2]),
793 + _placeholder("pattern_2", 1, 4, [-2]),
794 + ],
795 + "pattern_3": [
796 + _placeholder("pattern_3", 0, 5, [-3]),
797 + _placeholder("pattern_3", 1, 6, [-3]),
798 + ],
799 },
733 - ),
734 - ],
800 + },
801 + PromptReplacement: {
802 + 0: {},
803 + 1: {
804 + "pattern_1": [_placeholder("pattern_1", 0, 0, [-1])],
805 + "pattern_2": [_placeholder("pattern_2", 0, 2, [-2])],
806 + "pattern_3": [_placeholder("pattern_3", 0, 3, [-3])],
807 + },
808 + 2: {
809 + "pattern_1": [
810 + _placeholder("pattern_1", 0, 0, [-1]),
811 + _placeholder("pattern_1", 1, 1, [-1]),
812 + ],
813 + "pattern_2": [
814 + _placeholder("pattern_2", 0, 3, [-2]),
815 + _placeholder("pattern_2", 1, 4, [-2]),
816 + ],
817 + "pattern_3": [
818 + _placeholder("pattern_3", 0, 5, [-3]),
819 + _placeholder("pattern_3", 1, 6, [-3]),
820 + ],
821 + },
822 + },
823 + },
824 + {
825 + PromptInsertion: {
826 + 0: {},
827 + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
828 + 2: {
829 + "pattern_1": [
830 + _placeholder("pattern_1", 0, 1, [-1]),
831 + _placeholder("pattern_1", 1, 2, [-2]),
832 + ]
833 + },
834 + },
835 + PromptReplacement: {
836 + 0: {},
837 + 1: {"pattern_1": [_placeholder("pattern_1", 0, 0, [-1])]},
838 + 2: {
839 + "pattern_1": [
840 + _placeholder("pattern_1", 0, 0, [-1]),
841 + _placeholder("pattern_1", 1, 1, [-2]),
842 + ]
843 + },
844 + },
845 + },
846 + {
847 + PromptInsertion: {
848 + 0: {},
849 + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
850 + 2: {
851 + "pattern_1": [
852 + _placeholder("pattern_1", 0, 1, [-1]),
853 + _placeholder("pattern_1", 1, 2, [-2]),
854 + ]
855 + },
856 + },
857 + PromptReplacement: {
858 + 0: {},
859 + 1: {"pattern_1": [_placeholder("pattern_1", 0, 1, [-1])]},
860 + 2: {
861 + "pattern_1": [
862 + _placeholder("pattern_1", 0, 1, [-1]),
863 + _placeholder("pattern_1", 1, 2, [-2]),
864 + ]
865 + },
866 + },
867 + },
868 +]
869 +
870 +
871 +@pytest.mark.parametrize(
872 + ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
873 + FIND_UPDATE_TOKENS_TEST_CASES,
874 )
875 def test_find_update_tokens(
876 prompt,
@@ -769,6 +908,73 @@ def test_find_update_tokens(
908 assert new_prompt == expected
909
910
911 +@pytest.mark.parametrize(
912 + (
913 + "prompt",
914 + "target_by_key",
915 + "repl_by_key",
916 + "expected_by_update_type_mm_count",
917 + "expected_placeholders_by_update_type_mm_count",
918 + ),
919 + [
920 + (*case, placeholder_expected)
921 + for case, placeholder_expected in zip(
922 + FIND_UPDATE_TOKENS_TEST_CASES,
923 + FIND_UPDATE_TOKENS_PLACEHOLDER_EXPECTED,
924 + strict=True,
925 + )
926 + ],
927 +)
928 +def test_apply_token_matches_with_placeholders(
929 + prompt,
930 + target_by_key,
931 + repl_by_key,
932 + expected_by_update_type_mm_count,
933 + expected_placeholders_by_update_type_mm_count,
934 +):
935 + for update_type, expected_by_mm_count in expected_by_update_type_mm_count.items():
936 + for mm_count, expected in expected_by_mm_count.items():
937 + mm_prompt_updates = {
938 + key: [
939 + [update_type(key, target, repl_by_key[key]).resolve(i)]
940 + for i in range(mm_count)
941 + ]
942 + for key, target in target_by_key.items()
943 + }
944 +
945 + new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
946 + prompt,
947 + mm_prompt_updates,
948 + tokenizer=None,
949 + )
950 +
951 + if any(
952 + update_idx is None
953 + for update_idxs in result.values()
954 + for update_idx in update_idxs
955 + ):
956 + continue
957 +
958 + expected_placeholders = expected_placeholders_by_update_type_mm_count[
959 + update_type
960 + ][mm_count]
961 +
962 + # Only displayed on error
963 + print("update_type:", update_type)
964 + print("mm_count:", mm_count)
965 + print("mm_prompt_updates:", mm_prompt_updates)
966 + print("new_prompt:", new_prompt)
967 + print("result:", result)
968 + print("placeholders:", placeholders)
969 +
970 + assert new_prompt == expected
971 + assert {
972 + modality: ph_list
973 + for modality, ph_list in placeholders.items()
974 + if ph_list
975 + } == expected_placeholders
976 +
977 +
978 @pytest.mark.parametrize(
979 "repl_by_key",
980 [
vllm/model_executor/models/gemma3_mm.py
+34
@@ -373,6 +373,40 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
373
374 return token_ids, res
375
376 + def _apply_token_matches_with_placeholders(
377 + self,
378 + token_ids: list[int],
379 + mm_prompt_updates: MultiModalPromptUpdates,
380 + ) -> tuple[
381 + list[int],
382 + MultiModalPromptUpdatesApplyResult,
383 + Mapping[str, list[PlaceholderFeaturesInfo]],
384 + ]:
385 + new_token_ids, match_result = self._apply_token_matches(
386 + token_ids,
387 + mm_prompt_updates,
388 + )
389 +
390 + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
391 + modality: [] for modality in mm_prompt_updates
392 + }
393 +
394 + if all(
395 + all(update_idx is not None for update_idx in update_idxs)
396 + for update_idxs in match_result.values()
397 + ):
398 + placeholders = dict(
399 + self._find_mm_placeholders(
400 + new_token_ids,
401 + self._matched_updates_from_result(
402 + mm_prompt_updates,
403 + match_result,
404 + ),
405 + )
406 + )
407 +
408 + return new_token_ids, match_result, placeholders
409 +
410 def _find_mm_placeholders(
411 self,
412 new_token_ids: list[int],
vllm/model_executor/models/gemma3n_mm.py
+34
@@ -391,6 +391,40 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
391
392 return token_ids, res
393
394 + def _apply_token_matches_with_placeholders(
395 + self,
396 + token_ids: list[int],
397 + mm_prompt_updates: MultiModalPromptUpdates,
398 + ) -> tuple[
399 + list[int],
400 + MultiModalPromptUpdatesApplyResult,
401 + Mapping[str, list[PlaceholderFeaturesInfo]],
402 + ]:
403 + new_token_ids, match_result = self._apply_token_matches(
404 + token_ids,
405 + mm_prompt_updates,
406 + )
407 +
408 + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
409 + modality: [] for modality in mm_prompt_updates
410 + }
411 +
412 + if all(
413 + all(update_idx is not None for update_idx in update_idxs)
414 + for update_idxs in match_result.values()
415 + ):
416 + placeholders = dict(
417 + self._find_mm_placeholders(
418 + new_token_ids,
419 + self._matched_updates_from_result(
420 + mm_prompt_updates,
421 + match_result,
422 + ),
423 + )
424 + )
425 +
426 + return new_token_ids, match_result, placeholders
427 +
428 def _find_mm_placeholders(
429 self,
430 new_token_ids: list[int],
vllm/multimodal/processing/processor.py
+115 -25
@@ -958,6 +958,64 @@ def apply_token_matches(
958 return flatten_2d_lists(token_id_seqs), result
959
960
961 +def _apply_token_matches_with_placeholders(
962 + token_ids: list[int],
963 + mm_prompt_updates: "MultiModalPromptUpdates",
964 + tokenizer: TokenizerLike | None,
965 +) -> tuple[
966 + list[int],
967 + "MultiModalPromptUpdatesApplyResult",
968 + Mapping[str, list[PlaceholderFeaturesInfo]],
969 +]:
970 + matched_updates, result = _plan_prompt_updates(
971 + token_ids,
972 + mm_prompt_updates,
973 + tokenizer,
974 + )
975 + placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
976 + modality: [] for modality in mm_prompt_updates
977 + }
978 +
979 + new_token_ids = list[int]()
980 + prev_end_idx = 0
981 + for matched_update in matched_updates:
982 + update = matched_update.update
983 + match = matched_update.match
984 + matched_content = update.content.full
985 +
986 + if update.mode == UpdateMode.INSERT:
987 + end_idx_to_insert = match.end_idx
988 + elif update.mode == UpdateMode.REPLACE:
989 + end_idx_to_insert = match.start_idx
990 + else:
991 + assert_never(update.mode)
992 +
993 + new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
994 + start_idx = len(new_token_ids)
995 +
996 + tokens = _seq2tokens(tokenizer, matched_content)
997 + if tokens:
998 + content_is_embed = update.content.is_embed
999 + if content_is_embed is not None:
1000 + content_is_embed = content_is_embed(tokenizer, matched_content)
1001 +
1002 + placeholders[update.modality].append(
1003 + PlaceholderFeaturesInfo(
1004 + modality=update.modality,
1005 + item_idx=update.item_idx,
1006 + start_idx=start_idx,
1007 + tokens=tokens,
1008 + is_embed=content_is_embed,
1009 + )
1010 + )
1011 + new_token_ids.extend(tokens)
1012 +
1013 + prev_end_idx = match.end_idx
1014 +
1015 + new_token_ids.extend(token_ids[prev_end_idx:])
1016 + return new_token_ids, result, placeholders
1017 +
1018 +
1019 def apply_text_matches(
1020 prompt: str,
1021 mm_prompt_updates: "MultiModalPromptUpdates",
@@ -1671,6 +1729,22 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1729 tokenizer = self.info.get_tokenizer()
1730 return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1731
1732 + def _apply_token_matches_with_placeholders(
1733 + self,
1734 + token_ids: list[int],
1735 + mm_prompt_updates: MultiModalPromptUpdates,
1736 + ) -> tuple[
1737 + list[int],
1738 + MultiModalPromptUpdatesApplyResult,
1739 + Mapping[str, list[PlaceholderFeaturesInfo]],
1740 + ]:
1741 + tokenizer = self.info.get_tokenizer()
1742 + return _apply_token_matches_with_placeholders(
1743 + token_ids,
1744 + mm_prompt_updates,
1745 + tokenizer,
1746 + )
1747 +
1748 def _apply_text_matches(
1749 self,
1750 prompt: str,
@@ -1689,6 +1763,25 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1763 prompt, mm_prompt_updates, tokenizer
1764 )
1765
1766 + def _matched_updates_from_result(
1767 + self,
1768 + mm_prompt_updates: MultiModalPromptUpdates,
1769 + match_result: MultiModalPromptUpdatesApplyResult,
1770 + ) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]:
1771 + matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1772 + for modality, update_idxs in match_result.items():
1773 + for item_idx, update_idx in enumerate(update_idxs):
1774 + assert update_idx is not None, (
1775 + "Failed to apply prompt replacement for "
1776 + f"mm_items[{modality!r}][{item_idx}]"
1777 + )
1778 +
1779 + matched_updates[modality].append(
1780 + [mm_prompt_updates[modality][item_idx][update_idx]]
1781 + )
1782 +
1783 + return dict(matched_updates)
1784 +
1785 def _apply_prompt_updates(
1786 self,
1787 token_ids: list[int],
@@ -1697,11 +1790,24 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1790 """Apply multi-modal prompt updates to token IDs."""
1791 tokenizer = self.info.get_tokenizer()
1792
1700 - new_token_ids, match_result = self._apply_token_matches(
1701 - token_ids,
1702 - mm_prompt_updates,
1793 + new_token_ids, match_result, placeholders = (
1794 + self._apply_token_matches_with_placeholders(
1795 + token_ids,
1796 + mm_prompt_updates,
1797 + )
1798 )
1799
1800 + if all(
1801 + all(update_idx is not None for update_idx in update_idxs)
1802 + for update_idxs in match_result.values()
1803 + ):
1804 + placeholders = {
1805 + modality: modality_placeholders
1806 + for modality, modality_placeholders in placeholders.items()
1807 + if modality_placeholders
1808 + }
1809 + return new_token_ids, placeholders
1810 +
1811 # If the search text does not represent a special token,
1812 # it may have different token IDs in the prompt, because
1813 # the tokens may go across the boundaries of the search text.
@@ -1712,32 +1818,16 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1818 # Since it is inefficient to search for all possible tokenizations
1819 # of the search text in the prompt, we instead perform string-based
1820 # updates on the decoded token IDs, then encode them back.
1715 - if not all(
1716 - all(update_idx is not None for update_idx in update_idxs)
1717 - for update_idxs in match_result.values()
1718 - ):
1719 - new_text, match_result = self._apply_text_matches(
1720 - _seq2text(tokenizer, token_ids, use_cache=False),
1721 - mm_prompt_updates,
1722 - )
1723 -
1724 - new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1725 -
1726 - matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
1727 - for modality, update_idxs in match_result.items():
1728 - for item_idx, update_idx in enumerate(update_idxs):
1729 - assert update_idx is not None, (
1730 - "Failed to apply prompt replacement for "
1731 - f"mm_items[{modality!r}][{item_idx}]"
1732 - )
1821 + new_text, match_result = self._apply_text_matches(
1822 + _seq2text(tokenizer, token_ids, use_cache=False),
1823 + mm_prompt_updates,
1824 + )
1825
1734 - matched_updates[modality].append(
1735 - [mm_prompt_updates[modality][item_idx][update_idx]]
1736 - )
1826 + new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1827
1828 placeholders = self._find_mm_placeholders(
1829 new_token_ids,
1740 - dict(matched_updates),
1830 + self._matched_updates_from_result(mm_prompt_updates, match_result),
1831 )
1832
1833 return new_token_ids, placeholders