A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 08:30 UTC
[Bugfix] Fix O(n²) multimodal string prompt processing (#29667)
Signed-off-by: mertunsall <mertunsal1905@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Mert Unsal committed
Nov 28, 2025 at 16:10 UTC
c625d7b1c628bf8a49aa68d0a2628d95287be1ee
2 files changed
+65
-33
tests/multimodal/test_processing.py
+36
@@ -15,6 +15,7 @@ from vllm.multimodal.processing import (
15
PromptIndexTargets,
16
PromptInsertion,
17
PromptReplacement,
18
+ _apply_matches,
19
apply_text_matches,
20
apply_token_matches,
21
find_mm_placeholders,
@@ -1075,3 +1076,38 @@ def test_hf_processor_call_kwargs(
1076
1077
result = ctx.call_hf_processor(processor, {}, inference_kwargs)
1078
assert result == expected_kwargs
1079
+
1080
+
1081
+def test_apply_matches_no_match_exits_quickly():
1082
+ """
1083
+ Test that _apply_matches exits quickly when no matches are found.
1084
+
1085
+ Previously, _apply_matches had O(n²) behavior when no match was found
1086
+ because it would increment start_idx by 1 each iteration while
1087
+ re-scanning the entire prompt from prev_end_idx=0.
1088
+
1089
+ With the fix, it should exit immediately when no match is found.
1090
+ """
1091
+ import time
1092
+
1093
+ mock_tokenizer = cast(AnyTokenizer, object())
1094
+
1095
+ # Create a long prompt with no placeholder
1096
+ long_prompt = "x" * 10000
1097
+
1098
+ # Create update looking for a placeholder that doesn't exist
1099
+ mm_prompt_updates = {
1100
+ "image": [[PromptReplacement("image", "<image>", "REPLACED").resolve(0)]]
1101
+ }
1102
+
1103
+ start = time.perf_counter()
1104
+ result, _ = _apply_matches(
1105
+ long_prompt,
1106
+ mm_prompt_updates,
1107
+ mock_tokenizer,
1108
+ )
1109
+ elapsed = time.perf_counter() - start
1110
+
1111
+ # Should complete in < 100ms (was taking seconds before the fix)
1112
+ assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
1113
+ assert "".join(result) == long_prompt
vllm/multimodal/processing.py
+29
-33
@@ -742,7 +742,6 @@ def _apply_matches(
742
mm_prompt_updates: "MultiModalPromptUpdates",
743
tokenizer: AnyTokenizer,
744
) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
745
- prompt_len = len(prompt)
745
mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
746
747
out_seqs = list[str | list[int]]()
@@ -750,16 +749,15 @@ def _apply_matches(
749
m: [None] * len(items) for m, items in mm_prompt_updates.items()
750
}
751
752
+ # Early exit if no items to find
753
mm_found_counts = {
754
m: sum(r is not None for r in res) for m, res in out_result.items()
755
}
756
if _all_items_found(mm_item_counts, mm_found_counts):
757
return [prompt], out_result
758
759
- start_idx = prev_end_idx = 0
760
- while start_idx < max(prompt_len, 1): # Allow inserts into empty prompt
761
- found = False
762
-
759
+ prev_end_idx = 0
760
+ while True:
761
mode, matches_to_apply = _find_matches(
762
prompt,
763
mm_prompt_updates,
@@ -768,39 +766,37 @@ def _apply_matches(
766
current_result=out_result,
767
)
768
771
- if mode is not None:
772
- for (modality, item_idx), (match, update_idx) in matches_to_apply:
773
- found = True
769
+ if mode is None:
770
+ break # No more matches to find
771
775
- matched_update = mm_prompt_updates[modality][item_idx][update_idx]
776
- matched_content = matched_update.content.full
772
+ for (modality, item_idx), (match, update_idx) in matches_to_apply:
773
+ matched_update = mm_prompt_updates[modality][item_idx][update_idx]
774
+ matched_content = matched_update.content.full
775
778
- if mode == UpdateMode.INSERT:
779
- end_idx_to_insert = match.end_idx
780
- elif mode == UpdateMode.REPLACE:
781
- end_idx_to_insert = match.start_idx
782
- else:
783
- assert_never(mode)
784
-
785
- out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
786
- out_seqs.append(
787
- _seq2text(tokenizer, matched_content)
788
- if isinstance(prompt, str)
789
- else _seq2tokens(tokenizer, matched_content)
790
- )
791
- out_result[modality][item_idx] = update_idx
776
+ if mode == UpdateMode.INSERT:
777
+ end_idx_to_insert = match.end_idx
778
+ elif mode == UpdateMode.REPLACE:
779
+ end_idx_to_insert = match.start_idx
780
+ else:
781
+ assert_never(mode)
782
793
- # Exclude overlapping matches
794
- start_idx = prev_end_idx = match.end_idx
783
+ out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
784
+ out_seqs.append(
785
+ _seq2text(tokenizer, matched_content)
786
+ if isinstance(prompt, str)
787
+ else _seq2tokens(tokenizer, matched_content)
788
+ )
789
+ out_result[modality][item_idx] = update_idx
790
796
- mm_found_counts = {
797
- m: sum(r is not None for r in res) for m, res in out_result.items()
798
- }
799
- if _all_items_found(mm_item_counts, mm_found_counts):
800
- break
791
+ # Exclude overlapping matches
792
+ prev_end_idx = match.end_idx
793
802
- if not found:
803
- start_idx += 1
794
+ # Early exit if all items found
795
+ mm_found_counts = {
796
+ m: sum(r is not None for r in res) for m, res in out_result.items()
797
+ }
798
+ if _all_items_found(mm_item_counts, mm_found_counts):
799
+ break
800
801
out_seqs.append(prompt[prev_end_idx:])
802