[Perf] Speed up multimodal placeholder and token-match scanning (#50716)
Signed-off-by: Hareesh <hgsata@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Hareesh Gali committed
Aug 3, 2026 at 11:27 UTC
89ac407e3d8bdb34901dc63af71e4b5732de5ca0
2 files changed
+157
-44
tests/multimodal/test_processing.py
+60
@@ -1097,3 +1097,63 @@ def test_apply_matches_no_match_exits_quickly():
1097
# Should complete in < 100ms (was taking seconds before the fix)
1098
assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
1099
assert "".join(result) == long_prompt
1100
+
1101
+
1102
+def test_iter_token_matches_rejects_negative_start_idx():
1103
+ with pytest.raises(ValueError, match="non-negative"):
1104
+ list(iter_token_matches([1, 2, 3], [2], start_idx=-1))
1105
+
1106
+
1107
+def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
1108
+ """
1109
+ Test that placeholder scanning stays linear under adversarial candidates.
1110
+
1111
+ The fast-forward scan must not rescan the prompt tail per position when
1112
+ one candidate's first token never occurs (forcing a full search) while
1113
+ another's occurs at every position (forcing single-step advances).
1114
+ """
1115
+ prompt = [1] * 30_000
1116
+ mm_prompt_updates = {
1117
+ "absent": [[PromptReplacement("absent", [0], [999, 0]).resolve(0)]],
1118
+ "frequent_false_prefix": [
1119
+ [PromptReplacement("frequent_false_prefix", [0], [1, 2]).resolve(0)]
1120
+ ],
1121
+ }
1122
+
1123
+ start = time.perf_counter()
1124
+ result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
1125
+ elapsed = time.perf_counter() - start
1126
+
1127
+ assert result == {}
1128
+ assert elapsed < 0.5, f"find_mm_placeholders took {elapsed:.2f}s, expected < 0.5s"
1129
+
1130
+
1131
+@pytest.mark.parametrize(
1132
+ "prompt",
1133
+ [
1134
+ # Empty prompt: the scan loop is never entered
1135
+ [],
1136
+ # Non-empty prompt: the scan runs but never finds the first item,
1137
+ # so the second item must stay unresolved
1138
+ [1, 2, 3, 4, 5],
1139
+ ],
1140
+)
1141
+def test_find_mm_placeholders_resolves_content_lazily(prompt):
1142
+ """
1143
+ Test that content of items the scan never reaches is not resolved.
1144
+
1145
+ With `tokenizer=None`, resolving string content raises; the scan must
1146
+ return no placeholders instead of raising on the second item.
1147
+ """
1148
+ result = find_mm_placeholders(
1149
+ prompt,
1150
+ {
1151
+ "image": [
1152
+ [PromptReplacement("image", [0], [999]).resolve(0)],
1153
+ [PromptReplacement("image", [0], "never reached").resolve(1)],
1154
+ ]
1155
+ },
1156
+ tokenizer=None,
1157
+ )
1158
+
1159
+ assert result == {}
vllm/multimodal/processing/processor.py
+97
-44
@@ -627,13 +627,25 @@ def iter_token_matches(
627
628
Note that empty matches are ignored.
629
"""
630
+ if start_idx < 0:
631
+ raise ValueError("start_idx must be non-negative")
632
+
633
prompt_len = len(token_ids)
634
match_len = len(match_ids)
635
636
if match_len == 0:
637
return
638
636
- while start_idx < prompt_len - match_len + 1:
639
+ first_id = match_ids[0]
640
+ last_start_idx = prompt_len - match_len
641
+
642
+ while start_idx <= last_start_idx:
643
+ # Fast-forward to the next candidate position using a C-level scan
644
+ try:
645
+ start_idx = token_ids.index(first_id, start_idx, last_start_idx + 1)
646
+ except ValueError:
647
+ return
648
+
649
end_idx = start_idx + match_len
650
651
if token_ids[start_idx:end_idx] == match_ids:
@@ -708,6 +720,37 @@ def _find_matches(
720
mode: UpdateMode | None = None
721
mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()
722
723
+ # Items commonly share the same target object (a static PromptUpdate
724
+ # target resolves to the same object for every item); scan the prompt once
725
+ # per distinct target instead of once per item. The cache is keyed by
726
+ # object identity: it implies value equality, costs O(1) per lookup
727
+ # (hashing a long target value per lookup would defeat the point), and the
728
+ # target objects are kept alive by `mm_prompt_updates` for the duration of
729
+ # this call. Items with equal-valued but distinct target objects simply
730
+ # miss the cache and scan independently.
731
+ # NOTE: ResolvedPromptUpdate.iter_matches depends only on the target,
732
+ # prompt, tokenizer and start_idx, so updates sharing a target share the
733
+ # same first match within this invocation. Revisit this cache if matching
734
+ # ever depends on other update attributes.
735
+ first_match_by_target_id = dict[int, PromptTargetMatch | None]()
736
+
737
+ def get_first_match(update: "ResolvedPromptUpdate") -> PromptTargetMatch | None:
738
+ target = update.target
739
+ if isinstance(target, PromptIndex): # Matches are not cacheable
740
+ return next(
741
+ update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx),
742
+ None,
743
+ )
744
+
745
+ key = id(target)
746
+ if key not in first_match_by_target_id:
747
+ first_match_by_target_id[key] = next(
748
+ update.iter_matches(prompt, tokenizer, start_idx=prev_end_idx),
749
+ None,
750
+ )
751
+
752
+ return first_match_by_target_id[key]
753
+
754
for modality, modality_updates in mm_prompt_updates.items():
755
for item_idx, item_updates in enumerate(modality_updates):
756
if current_result[modality][item_idx] is not None:
@@ -717,19 +760,17 @@ def _find_matches(
760
if (modality, item_idx) in mm_matches:
761
break # Already found a match for this item
762
720
- for match in update.iter_matches(
721
- prompt,
722
- tokenizer,
723
- start_idx=prev_end_idx,
724
- ):
725
- # All matches should share the same mode
726
- if mode is None:
727
- mode = update.mode
728
- elif mode != update.mode:
729
- continue
763
+ match = get_first_match(update)
764
+ if match is None:
765
+ continue
766
+
767
+ # All matches should share the same mode
768
+ if mode is None:
769
+ mode = update.mode
770
+ elif mode != update.mode:
771
+ continue
772
731
- mm_matches[(modality, item_idx)] = match, update_idx
732
- break # Get only the first valid match per item
773
+ mm_matches[(modality, item_idx)] = match, update_idx
774
775
# Prioritize earlier matches
776
matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])
@@ -885,47 +926,59 @@ def _iter_placeholders(
926
prompt_len = len(prompt)
927
start_idx = 0
928
929
+ # The current (unfound) item's updates for each modality, with their
930
+ # content resolved to token ids; rebuilt whenever an item is found.
931
+ # Items are only resolved once the scan reaches them.
932
+ candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
933
+
934
while start_idx < prompt_len:
935
+ if candidates is None:
936
+ candidates = [
937
+ (modality, update, _seq2tokens(tokenizer, update.content.full))
938
+ for modality, modality_updates in mm_prompt_updates.items()
939
+ if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
940
+ for update in modality_updates[item_idx_by_modality[modality]]
941
+ ]
942
+ if not candidates:
943
+ return
944
+
945
found = False
946
+ first_token = prompt[start_idx]
947
+
948
+ for modality, update, content_tokens_full in candidates:
949
+ content_len_full = len(content_tokens_full)
950
+ end_idx_full = start_idx + content_len_full
951
891
- for modality, modality_updates in mm_prompt_updates.items():
892
- item_idx = item_idx_by_modality[modality]
893
- if item_idx >= mm_item_counts.get(modality, 0):
952
+ if content_len_full == 0 or end_idx_full > prompt_len:
953
continue
954
896
- for update in modality_updates[item_idx]:
955
+ # Check the first token before comparing the full slice
956
+ if (
957
+ first_token == content_tokens_full[0]
958
+ and prompt[start_idx:end_idx_full] == content_tokens_full
959
+ ):
960
content = update.content
898
- content_tokens_full = _seq2tokens(tokenizer, content.full)
899
- content_len_full = len(content_tokens_full)
900
- end_idx_full = start_idx + content_len_full
901
-
902
- if content_len_full == 0 or end_idx_full > prompt_len:
903
- continue
904
-
905
- if prompt[start_idx:end_idx_full] == content_tokens_full:
906
- content_is_embed = content.is_embed
907
- if content_is_embed is not None:
908
- content_is_embed = content_is_embed(tokenizer, content.full)
909
-
910
- yield PlaceholderFeaturesInfo(
911
- modality=modality,
912
- item_idx=item_idx,
913
- start_idx=start_idx,
914
- tokens=content_tokens_full,
915
- is_embed=content_is_embed,
916
- )
917
-
918
- # Exclude overlapping matches
919
- start_idx = end_idx_full
920
- item_idx_by_modality[modality] += 1
921
- found = True
922
- break
961
+ content_is_embed = content.is_embed
962
+ if content_is_embed is not None:
963
+ content_is_embed = content_is_embed(tokenizer, content.full)
964
+
965
+ yield PlaceholderFeaturesInfo(
966
+ modality=modality,
967
+ item_idx=item_idx_by_modality[modality],
968
+ start_idx=start_idx,
969
+ tokens=content_tokens_full,
970
+ is_embed=content_is_embed,
971
+ )
972
924
- if found:
973
+ # Exclude overlapping matches
974
+ start_idx = end_idx_full
975
+ item_idx_by_modality[modality] += 1
976
if _all_items_found(mm_item_counts, item_idx_by_modality):
977
return
978
928
- break # Go back to the outer while loop
979
+ candidates = None
980
+ found = True
981
+ break
982
983
if not found:
984
start_idx += 1