A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Refactor] Dynamic `target` and `content` for prompt updates (#23411)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 25, 2025 at 14:39 UTC
712d0f88d89940291b6a6a4db635c60187398acd
9 files changed
+472
-465
tests/multimodal/test_processing.py
+54
-58
@@ -17,13 +17,11 @@ from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
17
PromptReplacement, apply_text_matches,
18
apply_token_matches,
19
find_mm_placeholders,
20
- find_text_matches, find_token_matches,
20
iter_token_matches,
21
replace_token_matches)
22
# yapf: enable
23
from vllm.multimodal.profiling import MultiModalProfiler
24
from vllm.transformers_utils.tokenizer import AnyTokenizer
26
-from vllm.utils import full_groupby
25
26
from .utils import random_image
27
@@ -75,12 +73,15 @@ from .utils import random_image
73
),
74
],
75
)
76
+@pytest.mark.parametrize("start_idx", [0, 4, 8])
77
# yapf: enable
79
-def test_iter_token_matches(token_ids, match_ids, expected):
80
- result = list(iter_token_matches(token_ids, match_ids))
78
+def test_iter_token_matches(token_ids, match_ids, expected, start_idx):
79
+ result = list(iter_token_matches(token_ids, match_ids,
80
+ start_idx=start_idx))
81
82
# Manually constructed results
83
- assert [item._asdict() for item in result] == expected
83
+ assert [item._asdict() for item in result
84
+ ] == [item for item in expected if item["start_idx"] >= start_idx]
85
86
# Invariants
87
match_lens = [end - start for start, end in result]
@@ -241,21 +242,23 @@ def test_find_token_matches(
242
# Should not be used since there is nothing to convert to token IDs
243
mock_tokenizer = cast(AnyTokenizer, object())
244
244
- prompt_updates = [
245
- update_type(key, target, []).bind(mock_tokenizer)
245
+ prompt_updates = {
246
+ key: update_type(key, target, []).resolve(mock_tokenizer, 0)
247
for key, target in target_by_key.items()
247
- ]
248
- result = find_token_matches(prompt, prompt_updates)
248
+ }
249
+ result = {
250
+ key: list(update.iter_token_matches(prompt, mock_tokenizer))
251
+ for key, update in prompt_updates.items()
252
+ }
253
254
# Only displayed on error
255
print("result:", result)
256
257
# Manually constructed results
254
- result_groups = dict(full_groupby(result, key=lambda x: x.modality))
258
assert {
259
key: [
260
dict(start_idx=item.start_idx, end_idx=item.end_idx)
258
- for item in result_groups.get(key, [])
261
+ for item in result.get(key, [])
262
]
263
for key in expected_by_key
264
} == expected_by_key
@@ -388,21 +391,23 @@ def test_find_text_matches(
391
# Should not be used since there is nothing to convert to text
392
mock_tokenizer = cast(AnyTokenizer, object())
393
391
- prompt_updates = [
392
- update_type(key, target, []).bind(mock_tokenizer)
394
+ prompt_updates = {
395
+ key: update_type(key, target, []).resolve(mock_tokenizer, 0)
396
for key, target in target_by_key.items()
394
- ]
395
- result = find_text_matches(prompt, prompt_updates)
397
+ }
398
+ result = {
399
+ key: list(update.iter_text_matches(prompt, mock_tokenizer))
400
+ for key, update in prompt_updates.items()
401
+ }
402
403
# Only displayed on error
404
print("result:", result)
405
406
# Manually constructed results
401
- result_groups = dict(full_groupby(result, key=lambda x: x.modality))
407
assert {
408
key: [
409
dict(start_idx=item.start_idx, end_idx=item.end_idx)
405
- for item in result_groups.get(key, [])
410
+ for item in result.get(key, [])
411
]
412
for key in expected_by_key
413
} == expected_by_key
@@ -552,39 +557,37 @@ def test_find_update_text(
557
update_type,
558
expected_by_mm_count,
559
) in expected_by_update_type_mm_count.items():
555
- mm_prompt_updates = {
556
- key:
557
- [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
558
- for key, target in target_by_key.items()
559
- }
560
- mm_matches = {
561
- key: find_text_matches(prompt, updates)
562
- for key, updates in mm_prompt_updates.items()
563
- }
564
-
560
for mm_count, expected in expected_by_mm_count.items():
566
- result = apply_text_matches(
561
+ mm_prompt_updates = {
562
+ key: [[
563
+ update_type(key, target,
564
+ repl_by_key[key]).resolve(mock_tokenizer, i)
565
+ ] for i in range(mm_count)]
566
+ for key, target in target_by_key.items()
567
+ }
568
+
569
+ new_prompt, result = apply_text_matches(
570
prompt,
568
- mm_matches,
569
- {key: mm_count
570
- for key in repl_by_key},
571
+ mm_prompt_updates,
572
+ mock_tokenizer,
573
)
574
575
# Only displayed on error
576
print("update_type:", update_type)
577
print("mm_count:", mm_count)
576
- print("mm_matches:", mm_matches)
578
+ print("mm_prompt_updates:", mm_prompt_updates)
579
+ print("new_prompt:", new_prompt)
580
print("result:", result)
581
582
# Manually constructed results
580
- assert result == expected
583
+ assert new_prompt == expected
584
585
586
# yapf: disable
587
@pytest.mark.parametrize(
588
("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
589
[
587
- # Tokenized test cases of `test_find_replace_text`
590
+ # Tokenized test cases of `test_find_update_text`
591
# using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
592
(
593
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
@@ -726,32 +729,30 @@ def test_find_update_tokens(
729
update_type,
730
expected_by_mm_count,
731
) in expected_by_update_type_mm_count.items():
729
- mm_prompt_updates = {
730
- key:
731
- [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
732
- for key, target in target_by_key.items()
733
- }
734
- mm_matches = {
735
- key: find_token_matches(prompt, updates)
736
- for key, updates in mm_prompt_updates.items()
737
- }
738
-
732
for mm_count, expected in expected_by_mm_count.items():
740
- result = apply_token_matches(
733
+ mm_prompt_updates = {
734
+ key: [[
735
+ update_type(key, target,
736
+ repl_by_key[key]).resolve(mock_tokenizer, i)
737
+ ] for i in range(mm_count)]
738
+ for key, target in target_by_key.items()
739
+ }
740
+
741
+ new_prompt, result = apply_token_matches(
742
prompt,
742
- mm_matches,
743
- {key: mm_count
744
- for key in repl_by_key},
743
+ mm_prompt_updates,
744
+ mock_tokenizer,
745
)
746
747
# Only displayed on error
748
print("update_type:", update_type)
749
print("mm_count:", mm_count)
750
- print("mm_matches:", mm_matches)
750
+ print("mm_prompt_updates:", mm_prompt_updates)
751
+ print("new_prompt:", new_prompt)
752
print("result:", result)
753
754
# Manually constructed results
754
- assert result == expected
755
+ assert new_prompt == expected
756
757
758
# yapf: disable
@@ -878,17 +879,12 @@ def test_find_mm_placeholders(
879
mock_tokenizer = cast(AnyTokenizer, object())
880
881
mm_prompt_updates = {
881
- key: [update_type(key, [], repl).bind(mock_tokenizer)]
882
+ key: [[update_type(key, [], repl).resolve(mock_tokenizer, i)]
883
+ for i in range(3)]
884
for key, repl in repl_by_key.items()
885
}
886
885
- result = find_mm_placeholders(
886
- mm_prompt_updates,
887
- prompt,
888
- # Effectively match all occurrences in the prompt
889
- {key: 3
890
- for key in repl_by_key},
891
- )
887
+ result = find_mm_placeholders(prompt, mm_prompt_updates)
888
889
# Only displayed on error
890
print("result:", result)
vllm/model_executor/models/gemma3_mm.py
+12
-16
@@ -22,10 +22,12 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
22
MultiModalDataItems)
23
# yapf: disable
24
from vllm.multimodal.processing import (BaseMultiModalProcessor,
25
- BaseProcessingInfo, BoundPromptUpdate,
25
+ BaseProcessingInfo,
26
+ MultiModalPromptUpdates,
27
+ MultiModalPromptUpdatesApplyResult,
28
PlaceholderFeaturesInfo,
27
- PromptReplacement, PromptTargetMatch,
28
- PromptUpdate, PromptUpdateDetails,
29
+ PromptReplacement, PromptUpdate,
30
+ PromptUpdateDetails,
31
find_mm_placeholders,
32
replace_token_matches)
33
# yapf: enable
@@ -337,14 +339,10 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
339
def _apply_token_matches(
340
self,
341
prompt: list[int],
340
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
341
- mm_item_counts: Mapping[str, int],
342
- ) -> list[int]:
343
- token_ids = super()._apply_token_matches(
344
- prompt,
345
- mm_matches,
346
- mm_item_counts,
347
- )
342
+ mm_prompt_updates: MultiModalPromptUpdates,
343
+ ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
344
+ token_ids, res = super()._apply_token_matches(prompt,
345
+ mm_prompt_updates)
346
347
# "\n\n\n" and "\n\n\n\n" are single tokens
348
# Since our replacement can insert "\n\n" next to "\n"
@@ -373,13 +371,12 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
371
[newline_4],
372
)
373
376
- return token_ids
374
+ return token_ids, res
375
376
def _find_mm_placeholders(
377
self,
380
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
378
new_token_ids: list[int],
382
- mm_item_counts: Mapping[str, int],
379
+ mm_prompt_updates: MultiModalPromptUpdates,
380
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
381
# We need to detect "\n\n" inside "\n\n\n" and "\n\n\n\n"
382
tokenizer = self.info.get_tokenizer()
@@ -404,8 +401,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
401
repl_token_ids.extend(repl_toks)
402
repl_orig_idxs.extend(orig_idx for _ in range(len(repl_toks)))
403
407
- repls = find_mm_placeholders(mm_prompt_updates, repl_token_ids,
408
- mm_item_counts)
404
+ repls = find_mm_placeholders(repl_token_ids, mm_prompt_updates)
405
406
return {
407
modality: [
vllm/model_executor/models/gemma3n_mm.py
+12
-16
@@ -29,10 +29,12 @@ from vllm.multimodal.parse import (ImageProcessorItems, MultiModalDataItems,
29
MultiModalDataParser)
30
# yapf: disable
31
from vllm.multimodal.processing import (BaseMultiModalProcessor,
32
- BaseProcessingInfo, BoundPromptUpdate,
32
+ BaseProcessingInfo,
33
+ MultiModalPromptUpdates,
34
+ MultiModalPromptUpdatesApplyResult,
35
PlaceholderFeaturesInfo,
34
- PromptReplacement, PromptTargetMatch,
35
- PromptUpdate, PromptUpdateDetails,
36
+ PromptReplacement, PromptUpdate,
37
+ PromptUpdateDetails,
38
find_mm_placeholders,
39
replace_token_matches)
40
# yapf: enable
@@ -254,14 +256,10 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
256
def _apply_token_matches(
257
self,
258
prompt: list[int],
257
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
258
- mm_item_counts: Mapping[str, int],
259
- ) -> list[int]:
260
- token_ids = super()._apply_token_matches(
261
- prompt,
262
- mm_matches,
263
- mm_item_counts,
264
- )
259
+ mm_prompt_updates: MultiModalPromptUpdates,
260
+ ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
261
+ token_ids, res = super()._apply_token_matches(prompt,
262
+ mm_prompt_updates)
263
264
# "\n\n\n" and "\n\n\n\n" are single tokens
265
# Since our replacement can insert "\n\n" next to "\n"
@@ -290,13 +288,12 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
288
[newline_4],
289
)
290
293
- return token_ids
291
+ return token_ids, res
292
293
def _find_mm_placeholders(
294
self,
297
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
295
new_token_ids: list[int],
299
- mm_item_counts: Mapping[str, int],
296
+ mm_prompt_updates: MultiModalPromptUpdates,
297
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
298
# We need to detect "\n\n" inside "\n\n\n" and "\n\n\n\n"
299
tokenizer = self.info.get_tokenizer()
@@ -321,8 +318,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]
318
repl_token_ids.extend(repl_toks)
319
repl_orig_idxs.extend(orig_idx for _ in range(len(repl_toks)))
320
324
- repls = find_mm_placeholders(mm_prompt_updates, repl_token_ids,
325
- mm_item_counts)
321
+ repls = find_mm_placeholders(repl_token_ids, mm_prompt_updates)
322
323
return {
324
modality: [
vllm/model_executor/models/llava.py
+3
-10
@@ -828,26 +828,19 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
828
target=[image_token_id] * num_image_tokens,
829
replacement=get_replacement_mantis,
830
)
831
- ])
831
+ ], mm_item_counts)
832
833
prompt_ids, prompt, _ = self._apply_prompt_updates(
834
result["prompt_token_ids"],
835
mantis_mm_repls,
836
- mm_item_counts,
836
)
837
839
- unbound_orig_repls = self._get_prompt_updates(
838
+ orig_repls = self._get_mm_prompt_updates(
839
mm_items,
840
hf_processor_mm_kwargs,
841
mm_kwargs,
842
)
844
- orig_repls = self._bind_and_group_updates(unbound_orig_repls)
845
-
846
- mm_placeholders = self._find_mm_placeholders(
847
- orig_repls,
848
- prompt_ids,
849
- mm_item_counts,
850
- )
843
+ mm_placeholders = self._find_mm_placeholders(prompt_ids, orig_repls)
844
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
845
846
mm_placeholder_ranges = {
vllm/model_executor/models/phi3v.py
+6
-9
@@ -38,7 +38,8 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
38
# yapf conflicts with isort for this block
39
# yapf: disable
40
from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
- BaseProcessingInfo, BoundPromptUpdate,
41
+ BaseProcessingInfo,
42
+ MultiModalPromptUpdates,
43
PlaceholderFeaturesInfo,
44
PromptReplacement, PromptUpdate)
45
# yapf: enable
@@ -431,24 +432,21 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
432
433
return [_IMAGE_TOKEN_ID] * num_image_tokens
434
434
- num_images = mm_items.get_count("image", strict=False)
435
-
435
return [
436
PromptReplacement(
437
modality="image",
439
- target=image_token,
438
+ target=image_tokens.__getitem__,
439
replacement=get_replacement_phi3v,
441
- ) for image_token in image_tokens[:num_images]
440
+ )
441
]
442
443
def _apply_prompt_updates(
444
self,
445
token_ids: list[int],
447
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
448
- mm_item_counts: Mapping[str, int],
446
+ mm_prompt_updates: MultiModalPromptUpdates,
447
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
448
# align to hf behavior when there are images
451
- if len(mm_item_counts):
449
+ if len(mm_prompt_updates):
450
tokenizer = self.info.get_tokenizer()
451
# to decode token_ids to the original text, we need to
452
# 1. remove the first bos token
@@ -484,7 +482,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
482
token_ids, text, placeholders = super()._apply_prompt_updates(
483
token_ids=token_ids,
484
mm_prompt_updates=mm_prompt_updates,
487
- mm_item_counts=mm_item_counts,
485
)
486
487
# Keep the behavior in line with HF processor
vllm/model_executor/models/phi4_multimodal.py
+4
-8
@@ -1032,8 +1032,8 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1032
out_mm_kwargs: MultiModalKwargsItems,
1033
) -> Sequence[PromptUpdate]:
1034
tokenizer = self.info.get_tokenizer()
1035
- image_token_id = tokenizer.vocab[tokenizer.image_token]
1036
- audio_token_id = tokenizer.vocab[tokenizer.audio_token]
1035
+ image_token_id: int = tokenizer.vocab[tokenizer.image_token]
1036
+ audio_token_id: int = tokenizer.vocab[tokenizer.audio_token]
1037
1038
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1039
audio_processor = self.info.get_feature_extractor(
@@ -1053,9 +1053,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1053
processor=hf_processor,
1054
)
1055
1056
- image_tokens = [image_token_id] * num_image_tokens
1057
-
1058
- return image_tokens
1056
+ return [image_token_id] * num_image_tokens
1057
1058
def get_audio_replacement_phi4mm(item_idx: int):
1059
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -1066,9 +1064,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1064
audio_embed_size = self.info._compute_audio_embed_size(
1065
audio_frames)
1066
1069
- audio_tokens = [audio_token_id] * audio_embed_size
1070
-
1071
- return audio_tokens
1067
+ return [audio_token_id] * audio_embed_size
1068
1069
return [
1070
PromptReplacement(
vllm/model_executor/models/phi4mm.py
+7
-17
@@ -824,9 +824,7 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
824
processor=hf_processor,
825
)
826
827
- image_tokens = [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_image_tokens
828
-
829
- return image_tokens
827
+ return [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_image_tokens
828
829
def get_audio_replacement_phi4mm(item_idx: int):
830
audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -837,28 +835,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
835
audio_embed_size = self.info._compute_audio_embed_size(
836
audio_frames)
837
840
- audio_tokens = [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
841
-
842
- return audio_tokens
838
+ return [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
839
844
- num_images = mm_items.get_count("image", strict=False)
845
- num_audios = mm_items.get_count("audio", strict=False)
846
-
847
- image_repl = [
840
+ return [
841
PromptReplacement(
842
modality="image",
850
- target=image_token,
843
+ target=image_tokens.__getitem__,
844
replacement=get_image_replacement_phi4mm,
852
- ) for image_token in image_tokens[:num_images]
853
- ]
854
- audio_repl = [
845
+ ),
846
PromptReplacement(
847
modality="audio",
857
- target=audio_token,
848
+ target=audio_tokens.__getitem__,
849
replacement=get_audio_replacement_phi4mm,
859
- ) for audio_token in audio_tokens[:num_audios]
850
+ ),
851
]
861
- return image_repl + audio_repl
852
853
854
@MULTIMODAL_REGISTRY.register_processor(
vllm/model_executor/models/qwen2_5_omni_thinker.py
+1
-3
@@ -309,9 +309,8 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
309
310
if is_update_applied:
311
mm_placeholders = self._find_mm_placeholders(
312
- mm_prompt_updates,
312
prompt_ids,
314
- mm_item_counts,
313
+ mm_prompt_updates,
314
)
315
self._validate_mm_placeholders(
316
mm_placeholders,
@@ -328,7 +327,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
327
) = self._apply_prompt_updates(
328
prompt_ids,
329
mm_prompt_updates,
331
- mm_item_counts,
330
)
331
self._validate_mm_placeholders(
332
mm_placeholders,
vllm/multimodal/processing.py
+373
-328
@@ -44,10 +44,21 @@ PromptSeq = Union[str, list[int]]
44
"""A token sequence (list of token IDs) or text."""
45
46
47
+class _GetMatchIndex(Protocol):
48
+
49
+ def __call__(
50
+ self,
51
+ tokenizer: AnyTokenizer,
52
+ prompt: PromptSeq,
53
+ start_idx: int = 0,
54
+ ) -> Optional[int]:
55
+ ...
56
+
57
+
58
@dataclass
59
class PromptIndex:
60
"""Resolves to an index in the prompt."""
50
- get_match_index: Callable[[AnyTokenizer, PromptSeq], Optional[int]]
61
+ get_match_index: _GetMatchIndex
62
63
64
class PromptIndexTargets:
@@ -59,7 +70,7 @@ class PromptIndexTargets:
70
71
This results in a match even if the prompt is empty.
72
"""
62
- return PromptIndex(lambda tok, prompt: 0)
73
+ return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
74
75
@staticmethod
76
def prefix(seq: PromptSeq) -> PromptIndex:
@@ -70,7 +81,11 @@ class PromptIndexTargets:
81
def get_match_index(
82
tokenizer: AnyTokenizer,
83
prompt: PromptSeq,
84
+ start_idx: int = 0,
85
) -> Optional[int]:
86
+ if start_idx != 0:
87
+ return None
88
+
89
prefix = seq
90
91
if isinstance(prompt, str):
@@ -96,14 +111,24 @@ class PromptIndexTargets:
111
112
This results in a match even if the prompt is empty.
113
"""
99
- return PromptIndex(lambda tok, prompt: len(prompt))
114
+ return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
115
116
102
-PromptTarget = Union[PromptSeq, PromptIndex]
117
+UpdateTarget = Union[PromptSeq, PromptIndex]
118
"""
119
The token sequence or text to update.
120
"""
121
122
+PromptUpdateTarget = Union[Callable[[int], UpdateTarget], UpdateTarget]
123
+"""
124
+Given the index of the processed item within
125
+[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
126
+output the corresponding token sequence (or text).
127
+
128
+For convenience, you can directly pass in the token sequence (or text)
129
+instead of a function if it does not depend on the input.
130
+"""
131
+
132
133
@dataclass
134
class PromptUpdateDetails(Generic[_S]):
@@ -190,7 +215,7 @@ class PromptUpdate(ABC):
215
modality: str
216
"""The modality for which the update is made."""
217
193
- target: PromptTarget
218
+ target: PromptUpdateTarget
219
"""The token sequence (or text) to update."""
220
221
@property
@@ -205,10 +230,54 @@ class PromptUpdate(ABC):
230
"""Defines how to update the prompt."""
231
raise NotImplementedError
232
208
- def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptUpdate":
209
- return BoundPromptUpdate(
210
- _origin=self,
211
- tokenizer=tokenizer,
233
+ def _resolve_target(
234
+ self,
235
+ tokenizer: AnyTokenizer,
236
+ item_idx: int,
237
+ ) -> Union["_BoundPromptSequence", PromptIndex]:
238
+ target = self.target
239
+ if callable(target):
240
+ target = target(item_idx)
241
+
242
+ if isinstance(target, PromptIndex):
243
+ return target
244
+
245
+ return _BoundPromptSequence.from_seq(tokenizer, target)
246
+
247
+ def _resolve_content(
248
+ self,
249
+ tokenizer: AnyTokenizer,
250
+ item_idx: int,
251
+ ) -> "_BoundPromptContent":
252
+ content = self.content
253
+ if callable(content):
254
+ content = content(item_idx)
255
+
256
+ if not isinstance(content, PromptUpdateDetails):
257
+ content = PromptUpdateDetails.from_seq(content)
258
+
259
+ bound_full = _BoundPromptSequence.from_seq(tokenizer, content.full)
260
+ bound_content = _BoundPromptContent(full=bound_full,
261
+ is_embed=content.is_embed)
262
+
263
+ return bound_content
264
+
265
+ def resolve(
266
+ self,
267
+ tokenizer: AnyTokenizer,
268
+ item_idx: int,
269
+ ) -> "ResolvedPromptUpdate":
270
+ """
271
+ Given the index of the processed item within
272
+ [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
273
+ output a copy of this object with its lazy attributes resolved.
274
+ """
275
+ return ResolvedPromptUpdate(
276
+ modality=self.modality,
277
+ item_idx=item_idx,
278
+ mode=self.mode,
279
+ target=self._resolve_target(tokenizer, item_idx),
280
+ content=self._resolve_content(tokenizer, item_idx),
281
)
282
283
@@ -452,73 +521,90 @@ class _BoundPromptContent:
521
is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]]
522
523
455
-@dataclass
456
-class BoundPromptUpdate:
524
+class PromptTargetMatch(NamedTuple):
525
+ start_idx: int
526
+ end_idx: int
527
+
528
+
529
+@dataclass(frozen=True)
530
+class ResolvedPromptUpdate:
531
"""
458
- A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] bound
459
- to a tokenizer to automatically convert
460
- [`target`][vllm.multimodal.processing.PromptUpdate.target] and the result of
461
- [`get_content`][vllm.multimodal.processing.BoundPromptUpdate.get_content]
462
- between token sequence and text representations.
532
+ A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
533
+ lazy attributes resolved, apart from those related to tokenization.
534
"""
464
- _origin: PromptUpdate
465
- tokenizer: AnyTokenizer = field(repr=False)
535
467
- def __post_init__(self) -> None:
468
- self._content_cache = dict[int, _BoundPromptContent]()
536
+ modality: str
537
+ """The modality for which the update is made."""
538
470
- @property
471
- def modality(self) -> str:
472
- return self._origin.modality
539
+ item_idx: int
540
+ """The index within `modality` of the item this update pertains to."""
541
474
- @property
475
- def target(self) -> Union[_BoundPromptSequence, PromptIndex]:
476
- """The token sequence (or text) to update."""
477
- target = self._origin.target
542
+ mode: UpdateMode
543
+ """Defines how to update the prompt."""
544
479
- if isinstance(target, PromptIndex):
480
- return target
545
+ target: Union[_BoundPromptSequence, PromptIndex]
546
+ """The token sequence (or text) to update."""
547
482
- return _BoundPromptSequence.from_seq(self.tokenizer, target)
548
+ content: _BoundPromptContent = field(repr=False)
549
+ """The placeholder tokens that are part of the update."""
550
484
- @property
485
- def content(self) -> PromptUpdateContent:
486
- """The placeholder tokens that are part of the update."""
487
- return self._origin.content
551
+ def iter_token_matches(
552
+ self,
553
+ prompt: list[int],
554
+ tokenizer: AnyTokenizer,
555
+ *,
556
+ start_idx: int = 0,
557
+ ) -> Generator[PromptTargetMatch]:
558
+ """Yield each instance of `self.target` found in `prompt`."""
559
+ target = self.target
560
489
- @property
490
- def mode(self) -> UpdateMode:
491
- """Defines how to update the prompt."""
492
- return self._origin.mode
561
+ if isinstance(target, PromptIndex):
562
+ match_idx = target.get_match_index(tokenizer, prompt, start_idx)
563
+ if match_idx is not None:
564
+ yield PromptTargetMatch(match_idx, match_idx)
565
494
- def get_content(self, item_idx: int) -> _BoundPromptContent:
495
- """
496
- Given the index of the processed item within
497
- [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
498
- output the token sequence (or text) to update.
499
- """
500
- content = self.content
501
- if callable(content):
502
- cache_key = item_idx
503
- if cache_key in self._content_cache:
504
- return self._content_cache[cache_key]
566
+ return
567
506
- content = content(item_idx)
507
- else:
508
- cache_key = None
568
+ for match in iter_token_matches(prompt,
569
+ target.token_ids,
570
+ start_idx=start_idx):
571
+ yield PromptTargetMatch(match.start_idx, match.end_idx)
572
510
- if not isinstance(content, PromptUpdateDetails):
511
- content = PromptUpdateDetails.from_seq(content)
573
+ def iter_text_matches(
574
+ self,
575
+ prompt: str,
576
+ tokenizer: AnyTokenizer,
577
+ *,
578
+ start_idx: int = 0,
579
+ ) -> Generator[PromptTargetMatch]:
580
+ """Yield each instance of `self.target` found in `prompt`."""
581
+ target = self.target
582
513
- bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
514
- content.full)
515
- bound_content = _BoundPromptContent(full=bound_full,
516
- is_embed=content.is_embed)
583
+ if isinstance(target, PromptIndex):
584
+ match_idx = target.get_match_index(tokenizer, prompt, start_idx)
585
+ if match_idx is not None:
586
+ yield PromptTargetMatch(match_idx, match_idx)
587
518
- if cache_key is not None:
519
- self._content_cache[cache_key] = bound_content
588
+ return
589
521
- return bound_content
590
+ for match in re.finditer(re.escape(target.text), prompt,
591
+ pos=start_idx):
592
+ yield PromptTargetMatch(match.start(), match.end())
593
+
594
+ def iter_matches(
595
+ self,
596
+ prompt: Union[list[int], str],
597
+ tokenizer: AnyTokenizer,
598
+ *,
599
+ start_idx: int = 0,
600
+ ) -> Generator[PromptTargetMatch]:
601
+ """Yield each instance of `self.target` found in `prompt`."""
602
+ if isinstance(prompt, str):
603
+ return self.iter_text_matches(prompt,
604
+ tokenizer,
605
+ start_idx=start_idx)
606
+
607
+ return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
608
609
610
class _TokenMatch(NamedTuple):
@@ -529,6 +615,8 @@ class _TokenMatch(NamedTuple):
615
def iter_token_matches(
616
token_ids: list[int],
617
match_ids: list[int],
618
+ *,
619
+ start_idx: int = 0,
620
) -> Generator[_TokenMatch]:
621
"""
622
Yield each occurrence of `match_ids` in `token_ids`.
@@ -541,7 +629,6 @@ def iter_token_matches(
629
if match_len == 0:
630
return
631
544
- start_idx = 0
632
while start_idx < prompt_len - match_len + 1:
633
end_idx = start_idx + match_len
634
@@ -581,68 +668,6 @@ def replace_token_matches(
668
return flatten_2d_lists(out_seqs)
669
670
584
-@dataclass(repr=False)
585
-class PromptTargetMatch(ABC):
586
- _origin: BoundPromptUpdate
587
-
588
- @property
589
- def modality(self) -> str:
590
- return self._origin.modality
591
-
592
- @property
593
- @abstractmethod
594
- def start_idx(self) -> int:
595
- raise NotImplementedError
596
-
597
- @property
598
- @abstractmethod
599
- def end_idx(self) -> int:
600
- raise NotImplementedError
601
-
602
- def __repr__(self) -> str:
603
- return (f"{type(self).__name__}(modality={self.modality!r}, "
604
- f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
605
-
606
-
607
-@dataclass(repr=False)
608
-class _PromptTargetIndexMatch(PromptTargetMatch):
609
- match_idx: int
610
-
611
- @property
612
- def start_idx(self) -> int:
613
- return self.match_idx
614
-
615
- @property
616
- def end_idx(self) -> int:
617
- return self.match_idx
618
-
619
-
620
-@dataclass(repr=False)
621
-class _PromptTargetTokenMatch(PromptTargetMatch):
622
- match: _TokenMatch
623
-
624
- @property
625
- def start_idx(self) -> int:
626
- return self.match.start_idx
627
-
628
- @property
629
- def end_idx(self) -> int:
630
- return self.match.end_idx
631
-
632
-
633
-@dataclass(repr=False)
634
-class _PromptTargetTextMatch(PromptTargetMatch):
635
- match: re.Match[str]
636
-
637
- @property
638
- def start_idx(self) -> int:
639
- return self.match.start()
640
-
641
- @property
642
- def end_idx(self) -> int:
643
- return self.match.end()
644
-
645
-
671
@dataclass
672
class PlaceholderFeaturesInfo:
673
modality: str
@@ -665,163 +690,158 @@ class PlaceholderFeaturesInfo:
690
)
691
692
668
-def find_token_matches(
669
- prompt: list[int],
670
- prompt_updates: Sequence[BoundPromptUpdate],
671
-) -> Sequence[PromptTargetMatch]:
672
- """Return each target of `prompt_updates` found in `prompt`."""
673
-
674
- def get_matches(update: BoundPromptUpdate):
675
- target = update.target
676
-
677
- if isinstance(target, PromptIndex):
678
- match_idx = target.get_match_index(update.tokenizer, prompt)
679
- if match_idx is None:
680
- return []
681
-
682
- return [_PromptTargetIndexMatch(update, match_idx)]
693
+_MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]
694
684
- return [
685
- _PromptTargetTokenMatch(update, match)
686
- for match in iter_token_matches(prompt, target.token_ids)
687
- ]
688
-
689
- return [
690
- match for update in prompt_updates for match in get_matches(update)
691
- ]
692
-
693
-
694
-def find_text_matches(
695
- prompt: str,
696
- prompt_updates: Sequence[BoundPromptUpdate],
697
-) -> Sequence[PromptTargetMatch]:
698
- """Return each target of `prompt_updates` found in `prompt`."""
695
700
- def get_matches(update: BoundPromptUpdate):
701
- target = update.target
702
-
703
- if isinstance(target, PromptIndex):
704
- match_idx = target.get_match_index(update.tokenizer, prompt)
705
- if match_idx is None:
706
- return []
707
-
708
- return [_PromptTargetIndexMatch(update, match_idx)]
709
-
710
- return [
711
- _PromptTargetTextMatch(update, match)
712
- for match in re.finditer(re.escape(target.text), prompt)
713
- ]
714
-
715
- return [
716
- match for update in prompt_updates for match in get_matches(update)
717
- ]
718
-
719
-
720
-def _resolve_matches(
721
- prompt: PromptSeq,
722
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
723
-) -> list[PromptTargetMatch]:
724
- """
725
- Resolve `mm_matches` to ensure that there are no overlapping matches,
726
- and sort them such that earlier matches take priority over later ones.
727
- """
728
- matches = [m for matches in mm_matches.values() for m in matches]
729
-
730
- seen_matches: list[Optional[PromptTargetMatch]] = [None] * len(prompt)
731
-
732
- for match in matches:
733
- for idx in range(match.start_idx, match.end_idx):
734
- if seen_matches[idx] is not None:
735
- raise ValueError("Found overlapping matches "
736
- f"({seen_matches[idx]} and {match}) "
737
- f"at index={idx} of prompt={prompt}")
738
-
739
- seen_matches[idx] = match
740
-
741
- return sorted(matches, key=lambda x: x.start_idx)
696
+def _find_matches(
697
+ prompt: _S,
698
+ mm_prompt_updates: "MultiModalPromptUpdates",
699
+ tokenizer: AnyTokenizer,
700
+ *,
701
+ prev_end_idx: int = 0,
702
+ current_result: "MultiModalPromptUpdatesApplyResult",
703
+) -> tuple[Optional[UpdateMode], list[_MatchToApply]]:
704
+ mode: Optional[UpdateMode] = None
705
+ mm_matches = dict[tuple[str, int], tuple[PromptTargetMatch, int]]()
706
+
707
+ for modality, modality_updates in mm_prompt_updates.items():
708
+ for item_idx, item_updates in enumerate(modality_updates):
709
+ if current_result[modality][item_idx] is not None:
710
+ continue # Updates have already been applied for this item
711
+
712
+ for update_idx, update in enumerate(item_updates):
713
+ if (modality, item_idx) in mm_matches:
714
+ break # Already found a match for this item
715
+
716
+ for match in update.iter_matches(
717
+ prompt,
718
+ tokenizer,
719
+ start_idx=prev_end_idx,
720
+ ):
721
+ # All matches should share the same mode
722
+ if mode is None:
723
+ mode = update.mode
724
+ elif mode != update.mode:
725
+ continue
726
+
727
+ mm_matches[(modality, item_idx)] = match, update_idx
728
+ break # Get only the first valid match per item
729
+
730
+ # Prioritize earlier matches
731
+ matches_to_apply = sorted(mm_matches.items(), key=lambda item: item[1][0])
732
+
733
+ # To avoid conflicts, only replace one non-empty item at a time
734
+ if mode == UpdateMode.REPLACE:
735
+ matches_to_apply_ = list[_MatchToApply]()
736
+ has_non_empty_matches = False
737
+
738
+ for item in matches_to_apply:
739
+ _, (match, _) = item
740
+ if match.start_idx == match.end_idx:
741
+ matches_to_apply_.append(item)
742
+ elif not has_non_empty_matches:
743
+ has_non_empty_matches = True
744
+ matches_to_apply_.append(item)
745
+
746
+ matches_to_apply = matches_to_apply_
747
+
748
+ return mode, matches_to_apply
749
750
751
def _apply_matches(
752
prompt: _S,
746
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
747
- mm_item_counts: Mapping[str, int],
748
-) -> list[_S]:
749
- """Apply the updates in `mm_matches` to `prompt`."""
753
+ mm_prompt_updates: "MultiModalPromptUpdates",
754
+ tokenizer: AnyTokenizer,
755
+) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
756
+ prompt_len = len(prompt)
757
+
758
out_seqs = list[Union[str, list[int]]]()
751
- prev_end_idx = 0
752
- next_idx_by_modality = defaultdict[str, int](lambda: 0)
759
+ out_result: MultiModalPromptUpdatesApplyResult = {
760
+ m: [None] * len(items)
761
+ for m, items in mm_prompt_updates.items()
762
+ }
763
+
764
+ start_idx = prev_end_idx = 0
765
+ while start_idx < max(prompt_len, 1): # Allow inserts into empty prompt
766
+ found = False
767
754
- for match in _resolve_matches(prompt, mm_matches):
755
- modality = match.modality
768
+ mode, matches_to_apply = _find_matches(
769
+ prompt,
770
+ mm_prompt_updates,
771
+ tokenizer,
772
+ prev_end_idx=prev_end_idx,
773
+ current_result=out_result,
774
+ )
775
757
- item_start_idx = next_idx_by_modality[modality]
758
- max_item_count = mm_item_counts.get(modality, 0)
759
- if item_start_idx >= max_item_count:
760
- continue
776
+ if mode is not None:
777
+ for (modality, item_idx), (match, update_idx) in matches_to_apply:
778
+ found = True
779
762
- start_idx = match.start_idx
763
- end_idx = match.end_idx
764
- origin = match._origin
765
- mode = origin.mode
766
-
767
- if mode == UpdateMode.INSERT:
768
- out_seqs.append(prompt[prev_end_idx:end_idx])
769
- num_inserts = max_item_count
770
- elif mode == UpdateMode.REPLACE:
771
- out_seqs.append(prompt[prev_end_idx:start_idx])
772
- num_inserts = max_item_count if start_idx == end_idx else 1
773
- else:
774
- assert_never(mode)
780
+ matched_update = mm_prompt_updates[modality][item_idx][
781
+ update_idx]
782
+ matched_content = matched_update.content
783
776
- item_end_idx = min(item_start_idx + num_inserts, max_item_count)
784
+ if mode == UpdateMode.INSERT:
785
+ end_idx_to_insert = match.end_idx
786
+ elif mode == UpdateMode.REPLACE:
787
+ end_idx_to_insert = match.start_idx
788
+ else:
789
+ assert_never(mode)
790
778
- for item_idx in range(item_start_idx, item_end_idx):
779
- content = origin.get_content(item_idx)
780
- insert_seq = (content.full.text if isinstance(prompt, str) else
781
- content.full.token_ids)
791
+ out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
792
+ out_seqs.append(matched_content.full.text if isinstance(
793
+ prompt, str) else matched_content.full.token_ids)
794
+ out_result[modality][item_idx] = update_idx
795
783
- out_seqs.append(insert_seq)
796
+ # Exclude overlapping matches
797
+ start_idx = prev_end_idx = match.end_idx
798
785
- prev_end_idx = end_idx
786
- next_idx_by_modality[modality] += item_end_idx - item_start_idx
799
+ if not found:
800
+ start_idx += 1
801
802
out_seqs.append(prompt[prev_end_idx:])
803
790
- return cast(list[_S], out_seqs)
804
+ return cast(list[_S], out_seqs), out_result
805
806
807
def apply_token_matches(
808
prompt: list[int],
795
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
796
- mm_item_counts: Mapping[str, int],
797
-) -> list[int]:
798
- """Apply the updates in `mm_matches` to `prompt`."""
799
- if not mm_matches:
800
- return prompt
809
+ mm_prompt_updates: "MultiModalPromptUpdates",
810
+ tokenizer: AnyTokenizer,
811
+) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
812
+ """
813
+ Apply the updates in `mm_prompt_updates` to `prompt`.
814
802
- token_id_seqs = _apply_matches(prompt, mm_matches, mm_item_counts)
815
+ Matches are exclusive even when multiple modalities share
816
+ the same placeholder tokens. In that case, the modality that
817
+ appears earlier in `mm_prompt_updates` takes priority.
818
+ """
819
+ token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates,
820
+ tokenizer)
821
804
- return flatten_2d_lists(token_id_seqs)
822
+ return flatten_2d_lists(token_id_seqs), result
823
824
825
def apply_text_matches(
826
prompt: str,
809
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
810
- mm_item_counts: Mapping[str, int],
811
-) -> str:
812
- """Apply the updates in `mm_matches` to `prompt`."""
813
- if not mm_matches:
814
- return prompt
827
+ mm_prompt_updates: "MultiModalPromptUpdates",
828
+ tokenizer: AnyTokenizer,
829
+) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
830
+ """
831
+ Apply the updates in `mm_prompt_updates` to `prompt`.
832
816
- texts = _apply_matches(prompt, mm_matches, mm_item_counts)
833
+ Matches are exclusive even when multiple modalities share
834
+ the same placeholder tokens. In that case, the modality that
835
+ appears earlier in `mm_prompt_updates` takes priority.
836
+ """
837
+ texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
838
818
- return "".join(texts)
839
+ return "".join(texts), result
840
841
842
def _iter_placeholders(
822
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
843
prompt: list[int],
824
- mm_item_counts: Mapping[str, int],
844
+ mm_prompt_updates: "MultiModalPromptUpdates",
845
) -> Iterable[PlaceholderFeaturesInfo]:
846
"""
847
Yield each set of placeholder tokens found in `prompt`.
@@ -833,6 +853,8 @@ def _iter_placeholders(
853
Note that empty matches are ignored.
854
"""
855
prompt_len = len(prompt)
856
+ mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}
857
+
858
item_idx_by_modality = defaultdict[str, int](lambda: 0)
859
860
start_idx = 0
@@ -844,8 +866,8 @@ def _iter_placeholders(
866
if item_idx >= mm_item_counts.get(modality, 0):
867
continue
868
847
- for update_info in modality_updates:
848
- content = update_info.get_content(item_idx)
869
+ for update in modality_updates[item_idx]:
870
+ content = update.content
871
content_tokens_full = content.full.token_ids
872
content_len_full = len(content_tokens_full)
873
end_idx_full = start_idx + content_len_full
@@ -880,11 +902,10 @@ def _iter_placeholders(
902
903
904
def find_mm_placeholders(
883
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
905
prompt: list[int],
885
- mm_item_counts: Mapping[str, int],
906
+ mm_prompt_updates: "MultiModalPromptUpdates",
907
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
887
- it = _iter_placeholders(mm_prompt_updates, prompt, mm_item_counts)
908
+ it = _iter_placeholders(prompt, mm_prompt_updates)
909
return dict(full_groupby_modality(it))
910
911
@@ -989,12 +1010,20 @@ A collection of hashes with a similar structure as
1010
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1011
"""
1012
992
-MultiModalPromptUpdates = dict[str, Sequence[BoundPromptUpdate]]
1013
+MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1014
"""
1015
A collection of prompt updates with a similar structure as
1016
[`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
1017
"""
1018
1019
+MultiModalPromptUpdatesApplyResult = Mapping[str, list[Optional[int]]]
1020
+"""
1021
+For an item `MultiModalPromptUpdates[k][i]`,
1022
+`MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the
1023
+`ResolvedPromptUpdate` instance that has been applied, or `None` if none of the
1024
+`ResolvedPromptUpdate` instances have been applied.
1025
+"""
1026
+
1027
1028
class MultiModalProcessingInfo(NamedTuple):
1029
kwargs: MultiModalKwargsItems
@@ -1126,14 +1155,60 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1155
"""
1156
raise NotImplementedError
1157
1158
+ def _bind_and_group_updates(
1159
+ self,
1160
+ prompt_updates: Sequence[PromptUpdate],
1161
+ mm_item_counts: Mapping[str, int],
1162
+ ) -> MultiModalPromptUpdates:
1163
+ tokenizer = self.info.get_tokenizer()
1164
+
1165
+ return {
1166
+ modality:
1167
+ [[update.resolve(tokenizer, item_idx) for update in updates]
1168
+ for item_idx in range(mm_item_counts.get(modality, 0))]
1169
+ for modality, updates in full_groupby_modality(prompt_updates)
1170
+ }
1171
+
1172
+ def _get_mm_prompt_updates(
1173
+ self,
1174
+ mm_items: MultiModalDataItems,
1175
+ hf_processor_mm_kwargs: Mapping[str, object],
1176
+ out_mm_kwargs: MultiModalKwargsItems,
1177
+ ) -> MultiModalPromptUpdates:
1178
+ unbound_prompt_updates = self._get_prompt_updates(
1179
+ mm_items=mm_items,
1180
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1181
+ out_mm_kwargs=out_mm_kwargs,
1182
+ )
1183
+
1184
+ mm_prompt_updates = self._bind_and_group_updates(
1185
+ unbound_prompt_updates,
1186
+ mm_items.get_all_counts(),
1187
+ )
1188
+
1189
+ for modality, prompt_updates in mm_prompt_updates.items():
1190
+ for item_idx, item_prompt_updates in enumerate(prompt_updates):
1191
+ if len(item_prompt_updates) > 1:
1192
+ logger.warning_once(
1193
+ "Detected %d prompt updates for `mm_items[%r][%s]`. "
1194
+ "Multiple prompt updates per item is now "
1195
+ "deprecated and may be removed in v0.13. "
1196
+ "Instead, please specify dynamic update targets "
1197
+ "in the same prompt update definition by passing "
1198
+ "a function to `PromptUpdate.target`.",
1199
+ len(prompt_updates),
1200
+ modality,
1201
+ item_idx,
1202
+ )
1203
+
1204
+ return mm_prompt_updates
1205
+
1206
def _find_mm_placeholders(
1207
self,
1131
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1208
new_token_ids: list[int],
1133
- mm_item_counts: Mapping[str, int],
1209
+ mm_prompt_updates: MultiModalPromptUpdates,
1210
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1135
- return find_mm_placeholders(mm_prompt_updates, new_token_ids,
1136
- mm_item_counts)
1211
+ return find_mm_placeholders(new_token_ids, mm_prompt_updates)
1212
1213
def _get_hf_mm_data(
1214
self,
@@ -1421,13 +1496,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1496
mm_hashes = self._hash_mm_items(mm_data_items, hf_processor_mm_kwargs,
1497
tokenization_kwargs)
1498
1424
- unbound_prompt_updates = self._get_prompt_updates(
1499
+ mm_prompt_updates = self._get_mm_prompt_updates(
1500
mm_data_items,
1501
hf_processor_mm_kwargs,
1502
mm_kwargs,
1503
)
1429
- mm_prompt_updates = self._bind_and_group_updates(
1430
- unbound_prompt_updates)
1504
1505
mm_info = MultiModalProcessingInfo(
1506
kwargs=mm_kwargs,
@@ -1497,13 +1570,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1570
mm_missing_kwargs=mm_missing_kwargs,
1571
)
1572
1500
- unbound_prompt_updates = self._get_prompt_updates(
1573
+ mm_prompt_updates = self._get_mm_prompt_updates(
1574
mm_data_items,
1575
hf_processor_mm_kwargs,
1576
mm_kwargs,
1577
)
1505
- mm_prompt_updates = self._bind_and_group_updates(
1506
- unbound_prompt_updates)
1578
1579
mm_info = MultiModalProcessingInfo(
1580
kwargs=mm_kwargs,
@@ -1513,47 +1584,33 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1584
1585
return prompt_ids, mm_info, is_update_applied
1586
1516
- def _bind_and_group_updates(
1517
- self,
1518
- prompt_updates: Sequence[PromptUpdate],
1519
- ) -> dict[str, Sequence[BoundPromptUpdate]]:
1520
- tokenizer = self.info.get_tokenizer()
1521
-
1522
- it = (update.bind(tokenizer) for update in prompt_updates)
1523
- return dict(full_groupby_modality(it))
1524
-
1587
def _apply_token_matches(
1588
self,
1589
prompt: list[int],
1528
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
1529
- mm_item_counts: Mapping[str, int],
1530
- ) -> list[int]:
1531
- return apply_token_matches(prompt, mm_matches, mm_item_counts)
1590
+ mm_prompt_updates: MultiModalPromptUpdates,
1591
+ ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
1592
+ tokenizer = self.info.get_tokenizer()
1593
+ return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1594
1595
def _apply_text_matches(
1596
self,
1597
prompt: str,
1536
- mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
1537
- mm_item_counts: Mapping[str, int],
1538
- ) -> str:
1539
- return apply_text_matches(prompt, mm_matches, mm_item_counts)
1598
+ mm_prompt_updates: MultiModalPromptUpdates,
1599
+ ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
1600
+ tokenizer = self.info.get_tokenizer()
1601
+ return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1602
1603
def _apply_prompt_updates(
1604
self,
1605
token_ids: list[int],
1544
- mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1545
- mm_item_counts: Mapping[str, int],
1606
+ mm_prompt_updates: MultiModalPromptUpdates,
1607
) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1608
tokenizer = self.info.get_tokenizer()
1609
1549
- mm_token_matches = {
1550
- modality: find_token_matches(token_ids, updates)
1551
- for modality, updates in mm_prompt_updates.items()
1552
- }
1553
- mm_match_counts = {
1554
- modality: len(matches)
1555
- for modality, matches in mm_token_matches.items()
1556
- }
1610
+ new_token_ids, match_result = self._apply_token_matches(
1611
+ token_ids,
1612
+ mm_prompt_updates,
1613
+ )
1614
1615
# If the search text does not represent a special token,
1616
# it may have different token IDs in the prompt, because
@@ -1566,48 +1623,38 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1623
# of the search text in the prompt, we instead perform string-based
1624
# updates on the decoded token IDs, then encode them back.
1625
if all(
1569
- mm_match_counts.get(modality, 0) >= item_count
1570
- for modality, item_count in mm_item_counts.items()
1571
- ): # yapf: disable
1572
- token_ids = self._apply_token_matches(
1573
- token_ids,
1574
- mm_token_matches,
1575
- mm_item_counts,
1626
+ all(update_idx is not None for update_idx in update_idxs)
1627
+ for update_idxs in match_result.values()):
1628
+ new_text = decode_tokens(tokenizer, new_token_ids)
1629
+ else:
1630
+ new_text, match_result = self._apply_text_matches(
1631
+ decode_tokens(tokenizer, token_ids),
1632
+ mm_prompt_updates,
1633
)
1634
1578
- text = decode_tokens(tokenizer, token_ids)
1579
- matched_updates = {
1580
- modality: [match._origin for match in token_matches]
1581
- for modality, token_matches in mm_token_matches.items()
1582
- }
1583
- else:
1584
- text = decode_tokens(tokenizer, token_ids)
1585
-
1586
- mm_text_matches = {
1587
- modality: find_text_matches(text, updates)
1588
- for modality, updates in mm_prompt_updates.items()
1589
- }
1590
- text = self._apply_text_matches(
1591
- text,
1592
- mm_text_matches,
1593
- mm_item_counts,
1635
+ new_token_ids = encode_tokens(
1636
+ tokenizer,
1637
+ new_text,
1638
+ add_special_tokens=False,
1639
)
1640
1596
- token_ids = encode_tokens(tokenizer,
1597
- text,
1598
- add_special_tokens=False)
1599
- matched_updates = {
1600
- modality: [match._origin for match in token_matches]
1601
- for modality, token_matches in mm_text_matches.items()
1602
- }
1641
+ matched_updates = defaultdict[
1642
+ str, list[Sequence[ResolvedPromptUpdate]]](list)
1643
+ for modality, update_idxs in match_result.items():
1644
+ for item_idx, update_idx in enumerate(update_idxs):
1645
+ assert update_idx is not None, (
1646
+ "Failed to apply prompt replacement for "
1647
+ f"mm_items[{modality!r}][{item_idx}]")
1648
+
1649
+ matched_updates[modality].append(
1650
+ [mm_prompt_updates[modality][item_idx][update_idx]])
1651
1652
placeholders = self._find_mm_placeholders(
1605
- matched_updates,
1606
- token_ids,
1607
- mm_item_counts,
1653
+ new_token_ids,
1654
+ dict(matched_updates),
1655
)
1656
1610
- return token_ids, text, placeholders
1657
+ return new_token_ids, new_text, placeholders
1658
1659
def _validate_mm_kwargs(
1660
self,
@@ -1661,9 +1708,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1708
1709
if is_update_applied:
1710
mm_placeholders = self._find_mm_placeholders(
1664
- mm_prompt_updates,
1711
prompt_ids,
1666
- mm_item_counts,
1712
+ mm_prompt_updates,
1713
)
1714
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1715
@@ -1677,7 +1723,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1723
) = self._apply_prompt_updates(
1724
prompt_ids,
1725
mm_prompt_updates,
1680
- mm_item_counts,
1726
)
1727
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1728