@tduncan / Nvidia-Optimization-Stack-VLLM / commits / ebcd606467

[MM] Simplify prompt updates: replace `PromptSeq` with `list[int]` (#53372)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Qwen Code <noreply@alibaba-inc.com>

Cyrus Leung committed Aug 23, 2026 at 14:50 UTC ebcd60646741be239026db75358715ba707eb50f
53 files changed +830 -854
docs/contributing/model/multimodal.md
+6
@@ -532,6 +532,12 @@ return a list of [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instanc
532 Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies an update operation
533 (e.g.: insertion, replacement) performed by the HF processor.
534
535 +!!! note
536 + The target and content of each update are token sequences. When converting
537 + text to token sequences, remember to encode it with
538 + `cached_encode(..., add_special_tokens=False)`. Otherwise, the updated prompt
539 + may contain duplicated special tokens or fail to match the target.
540 +
541 === "Basic example: LLaVA"
542
543 Looking at HF's `LlavaProcessor`:
tests/models/multimodal/processing/test_openvla.py
+2 -2
@@ -192,11 +192,11 @@ def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
192 assert resolved.modality == "image"
193 assert [
194 (match.start_idx, match.end_idx)
195 - for match in resolved.iter_matches([1, 10, 11], _FakeTokenizer())
195 + for match in resolved.iter_token_matches([1, 10, 11])
196 ] == [(1, 1)]
197 assert content.full == [32000] * 256
198
199 - is_embed = content.is_embed(None, content.full)
199 + is_embed = content.is_embed(content.full)
200 assert is_embed.dtype == torch.bool
201 assert is_embed.tolist() == [True] * 256
202
tests/multimodal/test_cache.py
+2 -2
@@ -93,7 +93,7 @@ def test_cache_item_size(item, expected_size):
93 cache[""] = item
94 assert cache.currsize == expected_size
95
96 - prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
96 + prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
97
98 cache[""] = MultiModalProcessorCacheItem(item, [prompt_update])
99 assert cache.currsize == expected_size
@@ -151,7 +151,7 @@ def _compare_caches(
151 for item in all_items
152 ]
153
154 - prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
154 + prompt_update = PromptInsertion("dummy", [0], [1]).resolve(0)
155
156 for it in range(n_iter):
157 num_items_to_select = rng.randint(0, max_items_per_iter)
tests/multimodal/test_processing.py
+154 -341
@@ -12,18 +12,19 @@ from vllm.exceptions import VLLMValidationError
12 from vllm.multimodal import MULTIMODAL_REGISTRY
13 from vllm.multimodal.processing.context import InputProcessingContext
14 from vllm.multimodal.processing.processor import (
15 + BaseMultiModalProcessor,
16 PlaceholderFeaturesInfo,
17 PromptIndexTargets,
18 PromptInsertion,
19 PromptReplacement,
20 _apply_matches,
21 _apply_token_matches_with_placeholders,
21 - apply_text_matches,
22 apply_token_matches,
23 find_mm_placeholders,
24 iter_token_matches,
25 replace_token_matches,
26 )
27 +from vllm.utils.collection_utils import flatten_2d_lists
28
29 from .utils import random_image
30
@@ -242,7 +243,7 @@ def test_find_token_matches(
243 for key, target in target_by_key.items()
244 }
245 result = {
245 - key: list(update.iter_token_matches(prompt, tokenizer=None))
246 + key: list(update.iter_token_matches(prompt))
247 for key, update in prompt_updates.items()
248 }
249
@@ -259,315 +260,6 @@ def test_find_token_matches(
260 } == expected_by_key
261
262
262 -@pytest.mark.parametrize(
263 - ("prompt", "target_by_key", "expected_by_key"),
264 - [
265 - # Detokenized test cases of `test_find_token_matches`
266 - # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
267 - (
268 - "",
269 - {
270 - "pattern_1": "",
271 - "pattern_2": "<image>",
272 - "pattern_3": PromptIndexTargets.start(),
273 - "pattern_4": PromptIndexTargets.prefix("<image>"),
274 - "pattern_5": PromptIndexTargets.end(),
275 - },
276 - {
277 - "pattern_1": [{"start_idx": 0, "end_idx": 0}],
278 - "pattern_2": [],
279 - "pattern_3": [
280 - {"start_idx": 0, "end_idx": 0},
281 - ],
282 - "pattern_4": [],
283 - "pattern_5": [
284 - {"start_idx": 0, "end_idx": 0},
285 - ],
286 - },
287 - ),
288 - (
289 - "<image><image><image><image>",
290 - {
291 - "pattern_1": "<image>",
292 - "pattern_2": "<image><image>",
293 - "pattern_3": "<image><image><image>",
294 - "pattern_4": PromptIndexTargets.start(),
295 - "pattern_5": PromptIndexTargets.prefix("<image>"),
296 - "pattern_6": PromptIndexTargets.end(),
297 - },
298 - {
299 - "pattern_1": [
300 - {"start_idx": 0, "end_idx": 7},
301 - {"start_idx": 7, "end_idx": 14},
302 - {"start_idx": 14, "end_idx": 21},
303 - {"start_idx": 21, "end_idx": 28},
304 - ],
305 - "pattern_2": [
306 - {"start_idx": 0, "end_idx": 14},
307 - {"start_idx": 14, "end_idx": 28},
308 - ],
309 - "pattern_3": [
310 - {"start_idx": 0, "end_idx": 21},
311 - ],
312 - "pattern_4": [
313 - {"start_idx": 0, "end_idx": 0},
314 - ],
315 - "pattern_5": [
316 - {"start_idx": 7, "end_idx": 7},
317 - ],
318 - "pattern_6": [
319 - {"start_idx": 28, "end_idx": 28},
320 - ],
321 - },
322 - ),
323 - (
324 - "Image:<image><image><image>Image:<image><image>!",
325 - {
326 - "pattern_1": "Image:<image>",
327 - "pattern_2": "Image:<image><image><image>",
328 - "pattern_3": "Image:<unk><image>",
329 - "pattern_4": PromptIndexTargets.start(),
330 - "pattern_5": PromptIndexTargets.prefix("Image:<image>"),
331 - "pattern_6": PromptIndexTargets.end(),
332 - },
333 - {
334 - "pattern_1": [
335 - {"start_idx": 0, "end_idx": 13},
336 - {"start_idx": 27, "end_idx": 40},
337 - ],
338 - "pattern_2": [
339 - {"start_idx": 0, "end_idx": 27},
340 - ],
341 - "pattern_3": [],
342 - "pattern_4": [
343 - {"start_idx": 0, "end_idx": 0},
344 - ],
345 - "pattern_5": [
346 - {"start_idx": 13, "end_idx": 13},
347 - ],
348 - "pattern_6": [
349 - {"start_idx": 48, "end_idx": 48},
350 - ],
351 - },
352 - ),
353 - # Test regex escape
354 - (
355 - "<|image|><image><|image|><image>",
356 - {
357 - "pattern_1": "<|image|>",
358 - "pattern_2": "<|image|><image>",
359 - "pattern_3": "<|image|><image><|image|>",
360 - },
361 - {
362 - "pattern_1": [
363 - {"start_idx": 0, "end_idx": 9},
364 - {"start_idx": 16, "end_idx": 25},
365 - ],
366 - "pattern_2": [
367 - {"start_idx": 0, "end_idx": 16},
368 - {"start_idx": 16, "end_idx": 32},
369 - ],
370 - "pattern_3": [
371 - {"start_idx": 0, "end_idx": 25},
372 - ],
373 - },
374 - ),
375 - ],
376 -)
377 -@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
378 -def test_find_text_matches(
379 - prompt,
380 - target_by_key,
381 - expected_by_key,
382 - update_type,
383 -):
384 - prompt_updates = {
385 - key: update_type(key, target, []).resolve(0)
386 - for key, target in target_by_key.items()
387 - }
388 - result = {
389 - key: list(update.iter_text_matches(prompt, tokenizer=None))
390 - for key, update in prompt_updates.items()
391 - }
392 -
393 - # Only displayed on error
394 - print("result:", result)
395 -
396 - # Manually constructed results
397 - assert {
398 - key: [
399 - dict(start_idx=item.start_idx, end_idx=item.end_idx)
400 - for item in result.get(key, [])
401 - ]
402 - for key in expected_by_key
403 - } == expected_by_key
404 -
405 -
406 -@pytest.mark.parametrize(
407 - ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
408 - [
409 - (
410 - "Image:<image>Image:<image><image>!",
411 - {
412 - # We use `<image>` before `Image:` to test matches that
413 - # occur out of order
414 - "pattern_1": "<image>",
415 - "pattern_2": "Image:",
416 - "pattern_3": "!",
417 - },
418 - {
419 - # Test whether target is confused with replacement
420 - "pattern_1": "<image><image>",
421 - # Test empty replacement
422 - "pattern_2": "",
423 - # Test dynamic replacement (beyond the form of `unit * count`)
424 - "pattern_3": "?!?",
425 - },
426 - {
427 - PromptInsertion: {
428 - 0: "Image:<image>Image:<image><image>!",
429 - 1: "Image:<image><image><image>Image:<image><image>!?!?",
430 - 2: "Image:<image><image><image><image><image>Image:<image><image>!?!??!?", # noqa: E501
431 - },
432 - PromptReplacement: {
433 - 0: "Image:<image>Image:<image><image>!",
434 - 1: "<image><image>Image:<image><image>?!?",
435 - 2: "<image><image><image><image><image>?!?",
436 - },
437 - },
438 - ),
439 - # Test index targets
440 - (
441 - "",
442 - {
443 - "pattern_1": PromptIndexTargets.start(),
444 - "pattern_2": PromptIndexTargets.prefix("<image>"),
445 - "pattern_3": PromptIndexTargets.end(),
446 - },
447 - {
448 - "pattern_1": "1",
449 - "pattern_2": "2",
450 - "pattern_3": "3",
451 - },
452 - {
453 - PromptInsertion: {
454 - 0: "",
455 - 1: "13",
456 - 2: "1133",
457 - },
458 - PromptReplacement: {
459 - 0: "",
460 - 1: "13",
461 - 2: "1133",
462 - },
463 - },
464 - ),
465 - (
466 - "<image>",
467 - {
468 - "pattern_1": PromptIndexTargets.start(),
469 - "pattern_2": PromptIndexTargets.prefix("<image>"),
470 - "pattern_3": PromptIndexTargets.end(),
471 - },
472 - {
473 - "pattern_1": "1",
474 - "pattern_2": "2",
475 - "pattern_3": "3",
476 - },
477 - {
478 - PromptInsertion: {
479 - 0: "<image>",
480 - 1: "1<image>23",
481 - 2: "11<image>2233",
482 - },
483 - PromptReplacement: {
484 - 0: "<image>",
485 - 1: "1<image>23",
486 - 2: "11<image>2233",
487 - },
488 - },
489 - ),
490 - # Test different replacement per item
491 - (
492 - "<image><image><image>",
493 - {
494 - "pattern_1": "<image>",
495 - },
496 - {
497 - "pattern_1": lambda idx: str(idx + 1),
498 - },
499 - {
500 - PromptInsertion: {
501 - 0: "<image><image><image>",
502 - 1: "<image>1<image><image>",
503 - 2: "<image>12<image><image>",
504 - },
505 - PromptReplacement: {
506 - 0: "<image><image><image>",
507 - 1: "1<image><image>",
508 - 2: "12<image>",
509 - },
510 - },
511 - ),
512 - (
513 - "<image><image><image>",
514 - {
515 - "pattern_1": PromptIndexTargets.prefix("<image>"),
516 - },
517 - {
518 - "pattern_1": lambda idx: str(idx + 1),
519 - },
520 - {
521 - PromptInsertion: {
522 - 0: "<image><image><image>",
523 - 1: "<image>1<image><image>",
524 - 2: "<image>12<image><image>",
525 - },
526 - PromptReplacement: {
527 - 0: "<image><image><image>",
528 - 1: "<image>1<image><image>",
529 - 2: "<image>12<image><image>",
530 - },
531 - },
532 - ),
533 - ],
534 -)
535 -def test_find_update_text(
536 - prompt,
537 - target_by_key,
538 - repl_by_key,
539 - expected_by_update_type_mm_count,
540 -):
541 - for (
542 - update_type,
543 - expected_by_mm_count,
544 - ) in expected_by_update_type_mm_count.items():
545 - for mm_count, expected in expected_by_mm_count.items():
546 - mm_prompt_updates = {
547 - key: [
548 - [update_type(key, target, repl_by_key[key]).resolve(i)]
549 - for i in range(mm_count)
550 - ]
551 - for key, target in target_by_key.items()
552 - }
553 -
554 - new_prompt, result = apply_text_matches(
555 - prompt,
556 - mm_prompt_updates,
557 - tokenizer=None,
558 - )
559 -
560 - # Only displayed on error
561 - print("update_type:", update_type)
562 - print("mm_count:", mm_count)
563 - print("mm_prompt_updates:", mm_prompt_updates)
564 - print("new_prompt:", new_prompt)
565 - print("result:", result)
566 -
567 - # Manually constructed results
568 - assert new_prompt == expected
569 -
570 -
263 FIND_UPDATE_TOKENS_TEST_CASES = [
264 # Tokenized test cases of `test_find_update_text`
265 # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
@@ -891,11 +583,7 @@ def test_find_update_tokens(
583 for key, target in target_by_key.items()
584 }
585
894 - new_prompt, result = apply_token_matches(
895 - prompt,
896 - mm_prompt_updates,
897 - tokenizer=None,
898 - )
586 + new_prompt, result = apply_token_matches(prompt, mm_prompt_updates)
587
588 # Only displayed on error
589 print("update_type:", update_type)
@@ -945,7 +633,6 @@ def test_apply_token_matches_with_placeholders(
633 new_prompt, result, placeholders = _apply_token_matches_with_placeholders(
634 prompt,
635 mm_prompt_updates,
948 - tokenizer=None,
636 )
637
638 if any(
@@ -1097,7 +784,7 @@ def test_find_mm_placeholders(
784 for key, repl in repl_by_key.items()
785 }
786
1100 - result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
787 + result = find_mm_placeholders(prompt, mm_prompt_updates)
788
789 # Only displayed on error
790 print("result:", result)
@@ -1285,24 +972,18 @@ def test_apply_matches_no_match_exits_quickly():
972 With the fix, it should exit immediately when no match is found.
973 """
974 # Create a long prompt with no placeholder
1288 - long_prompt = "x" * 10000
975 + long_prompt = [1] * 10000
976
977 # Create update looking for a placeholder that doesn't exist
1291 - mm_prompt_updates = {
1292 - "image": [[PromptReplacement("image", "<image>", "REPLACED").resolve(0)]]
1293 - }
978 + mm_prompt_updates = {"image": [[PromptReplacement("image", [0], [-1]).resolve(0)]]}
979
980 start = time.perf_counter()
1296 - result, _ = _apply_matches(
1297 - long_prompt,
1298 - mm_prompt_updates,
1299 - tokenizer=None,
1300 - )
981 + result, _ = _apply_matches(long_prompt, mm_prompt_updates)
982 elapsed = time.perf_counter() - start
983
984 # Should complete in < 100ms (was taking seconds before the fix)
985 assert elapsed < 0.1, f"_apply_matches took {elapsed:.2f}s, expected < 0.1s"
1305 - assert "".join(result) == long_prompt
986 + assert flatten_2d_lists(result) == long_prompt
987
988
989 def test_apply_matches_many_shared_targets_scales_linearly():
@@ -1317,11 +998,7 @@ def test_apply_matches_many_shared_targets_scales_linearly():
998 prompt = [0] * item_count
999
1000 start = time.perf_counter()
1320 - result, match_result = apply_token_matches(
1321 - prompt,
1322 - mm_prompt_updates,
1323 - tokenizer=None,
1324 - )
1001 + result, match_result = apply_token_matches(prompt, mm_prompt_updates)
1002 elapsed = time.perf_counter() - start
1003
1004 assert len(result) == item_count * len(replacement)
@@ -1360,7 +1037,7 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
1037 }
1038
1039 start = time.perf_counter()
1363 - result = find_mm_placeholders(prompt, mm_prompt_updates, tokenizer=None)
1040 + result = find_mm_placeholders(prompt, mm_prompt_updates)
1041 elapsed = time.perf_counter() - start
1042
1043 assert result == {}
@@ -1377,22 +1054,158 @@ def test_find_mm_placeholders_avoids_quadratic_false_prefixes():
1054 [1, 2, 3, 4, 5],
1055 ],
1056 )
1380 -def test_find_mm_placeholders_resolves_content_lazily(prompt):
1057 +def test_find_mm_placeholders_stops_at_missing_item(prompt):
1058 """
1382 - Test that content of items the scan never reaches is not resolved.
1383 -
1384 - With `tokenizer=None`, resolving string content raises; the scan must
1385 - return no placeholders instead of raising on the second item.
1059 + Test that the scan returns no placeholders once it fails to find
1060 + an item's placeholder, leaving later items unresolved.
1061 """
1062 result = find_mm_placeholders(
1063 prompt,
1064 {
1065 "image": [
1066 [PromptReplacement("image", [0], [999]).resolve(0)],
1392 - [PromptReplacement("image", [0], "never reached").resolve(1)],
1067 + [PromptReplacement("image", [0], [998]).resolve(1)],
1068 ]
1069 },
1395 - tokenizer=None,
1070 )
1071
1072 assert result == {}
1073 +
1074 +
1075 +class _FakeTokenizer:
1076 + """
1077 + Character-level tokenizer where "foo" merges into one token differently
1078 + depending on whether it is followed by "d", like BPE merging "foo" in
1079 + "food" across the search-text boundary.
1080 + """
1081 +
1082 + _MERGES = {"food": (1000,), "foo": (101, 111, 111)}
1083 + _INVERSE = {ids: text for text, ids in _MERGES.items()}
1084 +
1085 + def encode(self, text: str, add_special_tokens: bool = True) -> list[int]:
1086 + token_ids = list[int]()
1087 + pos = 0
1088 + while pos < len(text):
1089 + for length in (4, 3):
1090 + word = text[pos : pos + length]
1091 + if word in self._MERGES:
1092 + token_ids.extend(self._MERGES[word])
1093 + pos += length
1094 + break
1095 + else:
1096 + token_ids.append(ord(text[pos]))
1097 + pos += 1
1098 + return token_ids
1099 +
1100 + def decode(
1101 + self,
1102 + token_ids: list[int],
1103 + skip_special_tokens: bool = False,
1104 + ) -> str:
1105 + chars = list[str]()
1106 + pos = 0
1107 + while pos < len(token_ids):
1108 + for length in (3, 1):
1109 + key = tuple(token_ids[pos : pos + length])
1110 + if key in self._INVERSE:
1111 + chars.append(self._INVERSE[key])
1112 + pos += length
1113 + break
1114 + else:
1115 + chars.append(chr(token_ids[pos]))
1116 + pos += 1
1117 + return "".join(chars)
1118 +
1119 +
1120 +class _FakeProcessingInfo:
1121 + def __init__(self, tokenizer) -> None:
1122 + self._tokenizer = tokenizer
1123 +
1124 + def get_tokenizer(self):
1125 + return self._tokenizer
1126 +
1127 +
1128 +class _TextFallbackProcessor(BaseMultiModalProcessor):
1129 + """Only `self.info.get_tokenizer()` is needed by the text fallback."""
1130 +
1131 + def __init__(self, tokenizer: _FakeTokenizer) -> None:
1132 + self.info = _FakeProcessingInfo(tokenizer)
1133 +
1134 + def _get_mm_fields_config(self, hf_inputs, hf_processor_mm_kwargs):
1135 + raise NotImplementedError
1136 +
1137 + def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs):
1138 + raise NotImplementedError
1139 +
1140 +
1141 +def _text_fallback_processor() -> BaseMultiModalProcessor:
1142 + return _TextFallbackProcessor(_FakeTokenizer())
1143 +
1144 +
1145 +def test_apply_prompt_updates_falls_back_to_text_matching():
1146 + """
1147 + Test that the fallback in `_apply_prompt_updates` finds targets that
1148 + tokenize differently inside the prompt ("foo" in "food").
1149 + """
1150 + processor = _text_fallback_processor()
1151 +
1152 + new_token_ids, placeholders = processor._apply_prompt_updates(
1153 + [1000], # "food"
1154 + {
1155 + "image": [
1156 + [PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)]
1157 + ]
1158 + },
1159 + )
1160 +
1161 + assert new_token_ids == [200, 201, ord("d")]
1162 + assert [p.to_range().offset for p in placeholders["image"]] == [0]
1163 + assert [p.tokens for p in placeholders["image"]] == [[200, 201]]
1164 +
1165 +
1166 +def test_apply_prompt_updates_falls_back_with_prefix_target():
1167 + """
1168 + Test that `PromptIndexTargets.prefix` targets are resolved against the
1169 + decoded text in the fallback path of `_apply_prompt_updates`.
1170 + """
1171 + processor = _text_fallback_processor()
1172 +
1173 + new_token_ids, placeholders = processor._apply_prompt_updates(
1174 + [1000], # "food"
1175 + {
1176 + "image": [
1177 + [
1178 + PromptInsertion(
1179 + "image",
1180 + PromptIndexTargets.prefix([101, 111, 111]),
1181 + [9],
1182 + ).resolve(0)
1183 + ]
1184 + ]
1185 + },
1186 + )
1187 +
1188 + assert new_token_ids == [101, 111, 111, 9, ord("d")]
1189 + assert [p.tokens for p in placeholders["image"]] == [[9]]
1190 +
1191 +
1192 +def test_apply_prompt_updates_falls_back_with_index_targets():
1193 + """
1194 + Test that the text resolvers of `PromptIndexTargets.start`/`end`
1195 + match against the decoded text when another item forces the
1196 + fallback in `_apply_prompt_updates`.
1197 + """
1198 + processor = _text_fallback_processor()
1199 +
1200 + new_token_ids, placeholders = processor._apply_prompt_updates(
1201 + [1000], # "food"
1202 + {
1203 + "image": [
1204 + [PromptReplacement("image", [101, 111, 111], [200, 201]).resolve(0)],
1205 + [PromptInsertion("image", PromptIndexTargets.end(), [9]).resolve(1)],
1206 + ]
1207 + },
1208 + )
1209 +
1210 + assert new_token_ids == [200, 201, ord("d"), 9]
1211 + assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
vllm/model_executor/models/audioflamingo3.py
+1 -1
@@ -483,7 +483,7 @@ class AudioFlamingo3MultiModalProcessor(
483 return [
484 PromptReplacement(
485 modality="audio",
486 - target=audio_token,
486 + target=[audio_token_id],
487 replacement=get_replacement_audioflamingo3,
488 )
489 ]
vllm/model_executor/models/cohere2_vision.py
+6 -2
@@ -40,6 +40,7 @@ from vllm.multimodal.processing import (
40 PromptReplacement,
41 PromptUpdate,
42 PromptUpdateDetails,
43 + cached_encode,
44 )
45 from vllm.sequence import IntermediateTensors
46 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -277,10 +278,12 @@ class Cohere2VisionMultiModalProcessor(
278 ) -> Sequence[PromptUpdate]:
279 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
280 image_token = hf_processor.image_token
281 + image_token_id = hf_processor.image_token_id
282 img_tokens_per_tile = int(hf_processor.patch_size**2)
283 img_line_break_token = hf_processor.img_line_break_token
284 boi_token = hf_processor.boi_token
285 eoi_token = hf_processor.eoi_token
286 + tokenizer = self.info.get_tokenizer()
287
288 def get_replacement(item_idx: int):
289 images = mm_items.get_items("image", ImageProcessorItems)
@@ -295,12 +298,13 @@ class Cohere2VisionMultiModalProcessor(
298 patch_tokens = image_token * img_tokens_per_tile + img_line_break_token
299 repl = f"{boi_token}{patch_tokens * num_patches}{eoi_token}"
300
298 - return PromptUpdateDetails.select_text(repl, image_token)
301 + repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
302 + return PromptUpdateDetails.select_token_id(repl_ids, image_token_id)
303
304 return [
305 PromptReplacement(
306 modality="image",
303 - target=image_token,
307 + target=[image_token_id],
308 replacement=get_replacement,
309 )
310 ]
vllm/model_executor/models/ernie45_vl.py
+9 -2
@@ -69,6 +69,7 @@ from vllm.multimodal.processing import (
69 BaseProcessingInfo,
70 PromptReplacement,
71 PromptUpdate,
72 + cached_encode,
73 )
74 from vllm.sequence import IntermediateTensors
75 from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1197,6 +1198,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
1198 out_mm_kwargs: MultiModalKwargsItems,
1199 ) -> Sequence[PromptUpdate]:
1200 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1201 + tokenizer = self.info.get_tokenizer()
1202
1203 before_placeholder = {
1204 "image": "<|image@placeholder|>",
@@ -1223,12 +1225,17 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
1225 )
1226 else:
1227 num_tokens = int(grid_thw.prod()) // merge_length
1226 - return after_placeholder[modality] * num_tokens
1228 + placeholder_ids = cached_encode(
1229 + tokenizer, after_placeholder[modality], add_special_tokens=False
1230 + )
1231 + return placeholder_ids * num_tokens
1232
1233 return [
1234 PromptReplacement(
1235 modality=modality,
1231 - target=before_placeholder[modality],
1236 + target=cached_encode(
1237 + tokenizer, before_placeholder[modality], add_special_tokens=False
1238 + ),
1239 replacement=partial(get_replacement_ernie45vl, modality=modality),
1240 )
1241 for modality in ("image", "video")
vllm/model_executor/models/funaudiochat.py
+1 -1
@@ -753,7 +753,7 @@ class FunAudioChatMultiModalProcessor(
753 return [
754 PromptReplacement(
755 modality="audio",
756 - target=audio_token,
756 + target=[audio_token_id],
757 replacement=get_replacement_funaudiochat,
758 )
759 ]
vllm/model_executor/models/gemma3_mm.py
+7 -4
@@ -32,6 +32,7 @@ from vllm.multimodal.processing.processor import (
32 PromptReplacement,
33 PromptUpdate,
34 PromptUpdateDetails,
35 + cached_encode,
36 replace_token_matches,
37 )
38 from vllm.sequence import IntermediateTensors
@@ -165,7 +166,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
166 image_height: int,
167 processor: Gemma3Processor,
168 mm_kwargs: Mapping[str, object],
168 - ) -> PromptUpdateDetails[str]:
169 + ) -> PromptUpdateDetails:
170 boi_token = processor.boi_token
171
172 num_crops = self.get_num_crops(
@@ -189,8 +190,9 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
190 tokenizer = processor.tokenizer
191 vocab = tokenizer.get_vocab()
192 image_token_id = vocab[tokenizer.image_token]
193 + repl_full_ids = cached_encode(tokenizer, repl_full, add_special_tokens=False)
194
193 - return PromptUpdateDetails.select_token_id(repl_full, image_token_id)
195 + return PromptUpdateDetails.select_token_id(repl_full_ids, image_token_id)
196
197 def get_num_image_tokens(
198 self,
@@ -313,7 +315,8 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
315 out_mm_kwargs: MultiModalKwargsItems,
316 ) -> Sequence[PromptUpdate]:
317 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
316 - image_token = hf_processor.boi_token
318 + tokenizer = self.info.get_tokenizer()
319 + boi_token_id = tokenizer.get_vocab()[hf_processor.boi_token]
320
321 def get_replacement_gemma3(item_idx: int):
322 images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,7 +332,7 @@ class Gemma3MultiModalProcessor(BaseMultiModalProcessor[Gemma3ProcessingInfo]):
332 return [
333 PromptReplacement(
334 modality="image",
332 - target=image_token,
335 + target=[boi_token_id],
336 replacement=get_replacement_gemma3,
337 )
338 ]
vllm/model_executor/models/gemma3n_mm.py
+17 -10
@@ -51,6 +51,7 @@ from vllm.multimodal.processing.processor import (
51 PromptReplacement,
52 PromptUpdate,
53 PromptUpdateDetails,
54 + cached_encode,
55 replace_token_matches,
56 )
57 from vllm.sequence import IntermediateTensors
@@ -183,31 +184,37 @@ class Gemma3nProcessingInfo(BaseProcessingInfo):
184 image_width: int,
185 image_height: int,
186 processor: Gemma3nProcessor,
186 - ) -> str:
187 + ) -> PromptUpdateDetails:
188 """
188 - Get the replacement text for image tokens.
189 + Get the replacement metadata for image tokens.
190
191 For Gemma3n, this should return the full_image_sequence which includes
192 BOI token, repeated image tokens, and EOI token.
193 """
194 + full_token_ids = cached_encode(
195 + processor.tokenizer, processor.full_image_sequence, add_special_tokens=False
196 + )
197 return PromptUpdateDetails.select_token_id(
194 - processor.full_image_sequence, processor.image_token_id
198 + full_token_ids, processor.image_token_id
199 )
200
201 def get_audio_repl(
202 self,
203 *,
204 processor: Gemma3nProcessor,
201 - ) -> str:
205 + ) -> PromptUpdateDetails:
206 """
203 - Get the replacement text for audio tokens.
207 + Get the replacement metadata for audio tokens.
208
209 For Gemma3n, this should return the full_audio_sequence which includes
210 BOA token, repeated audio tokens, and EOA token.
211 """
212 # Return the full audio sequence as defined by the processor
213 + full_token_ids = cached_encode(
214 + processor.tokenizer, processor.full_audio_sequence, add_special_tokens=False
215 + )
216 return PromptUpdateDetails.select_token_id(
210 - processor.full_audio_sequence, processor.audio_token_id
217 + full_token_ids, processor.audio_token_id
218 )
219
220
@@ -345,7 +352,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
352
353 # Handle image tokens
354 if "image" in mm_items:
348 - image_token = hf_processor.image_token
355 + image_token_id = hf_processor.image_token_id
356
357 def get_replacement_image(item_idx: int):
358 images = mm_items.get_items("image", ImageProcessorItems)
@@ -359,14 +366,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
366 prompt_updates.append(
367 PromptReplacement(
368 modality="image",
362 - target=image_token,
369 + target=[image_token_id],
370 replacement=get_replacement_image,
371 )
372 )
373
374 # Handle audio tokens
375 if "audio" in mm_items:
369 - audio_token = hf_processor.audio_token
376 + audio_token_id = hf_processor.audio_token_id
377
378 def get_replacement_audio(item_idx: int):
379 return self.info.get_audio_repl(
@@ -376,7 +383,7 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
383 prompt_updates.append(
384 PromptReplacement(
385 modality="audio",
379 - target=audio_token,
386 + target=[audio_token_id],
387 replacement=get_replacement_audio,
388 )
389 )
vllm/model_executor/models/gemma4_mm.py
+9 -9
@@ -332,7 +332,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
332 image_height: int,
333 processor: Gemma4Processor | None,
334 max_soft_tokens: int | None = None,
335 - ) -> PromptUpdateDetails[list[int]]:
335 + ) -> PromptUpdateDetails:
336 """Return the dynamic image token sequence for this image.
337
338 Computes the exact number of soft tokens the vision tower will
@@ -386,7 +386,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
386 *,
387 audio_len: int,
388 processor: Gemma4Processor | None,
389 - ) -> PromptUpdateDetails[list[int]]:
389 + ) -> PromptUpdateDetails:
390 """Return the dynamic audio token sequence for this audio.
391
392 Computes the number of soft tokens from the audio waveform
@@ -414,7 +414,7 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
414 timestamps: list[float],
415 num_soft_tokens_per_frame: list[int],
416 processor: Gemma4Processor,
417 - ) -> PromptUpdateDetails[list[int]]:
417 + ) -> PromptUpdateDetails:
418 """Build the full token replacement for one video.
419
420 Produces the same interleaved sequence as the HF Gemma4Processor:
@@ -818,7 +818,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
818 # one image_token exists per image in the token stream.
819 # The replacement expands it to the full image sequence
820 # (boi + N×image_token + eoi, where N = max_soft_tokens).
821 - image_token = hf_processor.image_token
821 + image_token_id = hf_processor.image_token_id
822
823 def get_replacement_image(item_idx: int):
824 images = mm_items.get_items("image", ImageProcessorItems)
@@ -848,13 +848,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
848 prompt_updates.append(
849 PromptReplacement(
850 modality="image",
851 - target=image_token,
851 + target=[image_token_id],
852 replacement=get_replacement_image,
853 )
854 )
855
856 if "video" in mm_items:
857 - video_token = hf_processor.video_token
857 + video_token_id = hf_processor.video_token_id
858
859 def get_replacement_video(item_idx: int):
860 out_item = out_mm_kwargs["video"][item_idx]
@@ -869,13 +869,13 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
869 prompt_updates.append(
870 PromptReplacement(
871 modality="video",
872 - target=video_token,
872 + target=[video_token_id],
873 replacement=get_replacement_video,
874 )
875 )
876
877 if "audio" in mm_items:
878 - audio_token = hf_processor.audio_token
878 + audio_token_id = hf_processor.audio_token_id
879
880 def get_replacement_audio(item_idx: int):
881 audios = mm_items.get_items("audio", AudioProcessorItems)
@@ -888,7 +888,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
888 prompt_updates.append(
889 PromptReplacement(
890 modality="audio",
891 - target=audio_token,
891 + target=[audio_token_id],
892 replacement=get_replacement_audio,
893 )
894 )
vllm/model_executor/models/glm4_1v.py
+11 -2
@@ -94,6 +94,7 @@ from vllm.multimodal.processing import (
94 PromptReplacement,
95 PromptUpdate,
96 PromptUpdateDetails,
97 + cached_encode,
98 )
99 from vllm.sequence import IntermediateTensors
100 from vllm.transformers_utils.processor import get_processor_cls_name_from_config
@@ -1694,6 +1695,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1695 ) -> Sequence[PromptUpdate]:
1696 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1697 image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
1698 + tokenizer = self.info.get_tokenizer()
1699 + vocab = tokenizer.get_vocab()
1700
1701 merge_length = image_processor.merge_size**2
1702
@@ -1722,12 +1725,18 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
1725 return [
1726 PromptReplacement(
1727 modality="image",
1725 - target=hf_processor.image_token,
1728 + target=cached_encode(
1729 + tokenizer, hf_processor.image_token, add_special_tokens=False
1730 + ),
1731 replacement=get_image_replacement,
1732 ),
1733 PromptReplacement(
1734 modality="video",
1730 - target="<|begin_of_video|><|video|><|end_of_video|>",
1735 + target=[
1736 + vocab["<|begin_of_video|>"],
1737 + vocab["<|video|>"],
1738 + vocab["<|end_of_video|>"],
1739 + ],
1740 replacement=get_video_replacement,
1741 ),
1742 ]
vllm/model_executor/models/glmasr.py
+1 -1
@@ -896,7 +896,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
896 return [
897 PromptReplacement(
898 modality="audio",
899 - target=audio_token,
899 + target=[audio_token_id],
900 replacement=get_replacement_glmasr,
901 )
902 ]
vllm/model_executor/models/h2ovl.py
+4 -1
@@ -25,6 +25,7 @@ from vllm.multimodal.processing.processor import (
25 ProcessorInputs,
26 PromptReplacement,
27 TimingContext,
28 + cached_encode,
29 )
30 from vllm.transformers_utils.processors.h2ovl import H2OVLImageProcessor, H2OVLProcessor
31
@@ -90,6 +91,8 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
91 hf_processor: H2OVLProcessor,
92 out_mm_data: BatchedTensorInputs,
93 ):
94 + tokenizer = self.info.get_tokenizer()
95 +
96 if "image_num_patches" in out_mm_data:
97 image_num_patches = out_mm_data["image_num_patches"]
98 assert isinstance(image_num_patches, torch.Tensor)
@@ -127,7 +130,7 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
130
131 return PromptReplacement(
132 modality="image",
130 - target="<image>",
133 + target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
134 replacement=get_replacement_internvl,
135 )
136
vllm/model_executor/models/idefics3.py
+10 -5
@@ -49,6 +49,7 @@ from vllm.multimodal.processing import (
49 PromptReplacement,
50 PromptUpdate,
51 PromptUpdateDetails,
52 + cached_encode,
53 )
54 from vllm.sequence import IntermediateTensors
55 from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -315,7 +316,8 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
316 out_mm_kwargs: MultiModalKwargsItems,
317 ) -> Sequence[PromptUpdate]:
318 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
318 - image_token, _, _ = self.info._get_image_token(hf_processor)
319 + image_token_id = hf_processor.image_token_id
320 + tokenizer = self.info.get_tokenizer()
321
322 def get_replacement_idefics3(item_idx: int) -> PromptUpdateDetails:
323 images = mm_items.get_items("image", ImageProcessorItems)
@@ -329,15 +331,18 @@ class Idefics3MultiModalProcessor(BaseMultiModalProcessor[Idefics3ProcessingInfo
331 mm_kwargs=hf_processor_mm_kwargs,
332 )
333
332 - return PromptUpdateDetails.select_text(
333 - image_repl,
334 - embed_text=image_token,
334 + image_repl_ids = cached_encode(
335 + tokenizer, image_repl, add_special_tokens=False
336 + )
337 + return PromptUpdateDetails.select_token_id(
338 + image_repl_ids,
339 + image_token_id,
340 )
341
342 return [
343 PromptReplacement(
344 modality="image",
340 - target=image_token,
345 + target=[image_token_id],
346 replacement=get_replacement_idefics3,
347 )
348 ]
vllm/model_executor/models/interns1.py
+22 -9
@@ -45,6 +45,7 @@ from vllm.multimodal.processing import (
45 PromptReplacement,
46 PromptUpdate,
47 PromptUpdateDetails,
48 + cached_encode,
49 )
50 from vllm.sequence import IntermediateTensors
51 from vllm.transformers_utils.processor import cached_video_processor_from_config
@@ -355,11 +356,9 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
356
357 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
358 tokenizer = hf_processor.tokenizer
358 - video_token_id = tokenizer.encode(
359 - hf_processor.video_token, add_special_tokens=False
360 - )
361 - assert len(video_token_id) == 1
362 - video_token_id = video_token_id[0]
359 + vocab = tokenizer.get_vocab()
360 +
361 + video_token_id = vocab[hf_processor.video_token]
362
363 prompt_text = re.sub(
364 hf_processor.image_token, "<image_placeholder>", prompt_text
@@ -469,10 +468,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
468 ) -> Sequence[PromptUpdate]:
469 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
470 img_context_token = hf_processor.image_token
471 + img_context_token_id = hf_processor.image_token_id
472 start_image_token = hf_processor.start_image_token
473 end_image_token = hf_processor.end_image_token
474 video_token = hf_processor.video_token
475
476 + tokenizer = self.info.get_tokenizer()
477 + video_token_id = cached_encode(tokenizer, video_token, add_special_tokens=False)
478 + assert len(video_token_id) == 1
479 + video_token_id = video_token_id[0]
480 +
481 out_mm_data = out_mm_kwargs.get_data()
482 if "video_num_patches" in out_mm_data:
483 video_num_patches = out_mm_data["video_num_patches"]
@@ -501,7 +506,12 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
506
507 repl_features = img_context_token * feature_size
508 repl_full = start_image_token + repl_features + end_image_token
504 - return PromptUpdateDetails.select_text(repl_full, img_context_token)
509 + repl_full_ids = cached_encode(
510 + tokenizer, repl_full, add_special_tokens=False
511 + )
512 + return PromptUpdateDetails.select_token_id(
513 + repl_full_ids, img_context_token_id
514 + )
515
516 def get_replacement_interns1_video(item_idx: int):
517 num_patches = video_num_patches[item_idx]
@@ -512,17 +522,20 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
522 [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
523 )
524
515 - return PromptUpdateDetails.select_text(repl_full, video_token)
525 + repl_full_ids = cached_encode(
526 + tokenizer, repl_full, add_special_tokens=False
527 + )
528 + return PromptUpdateDetails.select_token_id(repl_full_ids, video_token_id)
529
530 return [
531 PromptReplacement(
532 modality="image",
520 - target=img_context_token,
533 + target=[img_context_token_id],
534 replacement=get_replacement_interns1_image,
535 ),
536 PromptReplacement(
537 modality="video",
525 - target=video_token,
538 + target=[video_token_id],
539 replacement=get_replacement_interns1_video,
540 ),
541 ]
vllm/model_executor/models/internvl.py
+7 -2
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
43 BaseProcessingInfo,
44 PromptReplacement,
45 PromptUpdate,
46 + cached_encode,
47 )
48 from vllm.sequence import IntermediateTensors
49 from vllm.transformers_utils.processors.internvl import (
@@ -264,6 +265,8 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
265 hf_processor: InternVLProcessor,
266 out_mm_data: BatchedTensorInputs,
267 ):
268 + tokenizer = self.info.get_tokenizer()
269 +
270 if "image_num_patches" in out_mm_data:
271 image_num_patches = out_mm_data["image_num_patches"]
272 assert isinstance(image_num_patches, torch.Tensor)
@@ -298,7 +301,7 @@ class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
301
302 return PromptReplacement(
303 modality="image",
301 - target="<image>",
304 + target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
305 replacement=get_replacement_internvl,
306 )
307
@@ -498,6 +501,8 @@ class InternVLMultiModalProcessor(
501 hf_processor: InternVLProcessor,
502 out_mm_data: BatchedTensorInputs,
503 ):
504 + tokenizer = self.info.get_tokenizer()
505 +
506 if "video_num_patches" in out_mm_data:
507 video_num_patches = out_mm_data["video_num_patches"]
508 assert isinstance(video_num_patches, torch.Tensor)
@@ -514,7 +519,7 @@ class InternVLMultiModalProcessor(
519
520 return PromptReplacement(
521 modality="video",
517 - target="<video>",
522 + target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
523 replacement=get_video_replacement_internvl,
524 )
525
vllm/model_executor/models/isaac.py
+8 -3
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
54 PromptReplacement,
55 PromptUpdate,
56 PromptUpdateDetails,
57 + cached_encode,
58 )
59 from vllm.sequence import IntermediateTensors
60 from vllm.tokenizers import cached_tokenizer_from_config
@@ -450,6 +451,10 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
451 out_mm_kwargs: MultiModalKwargsItems,
452 ) -> Sequence[PromptUpdate]:
453 image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
454 + tokenizer = self.info.get_tokenizer()
455 + image_pad_token_ids = cached_encode(
456 + tokenizer, "<|image_pad|>", add_special_tokens=False
457 + )
458
459 pixel_shuffle_scale = getattr(image_processor, "pixel_shuffle_scale", 2)
460 merge_length = pixel_shuffle_scale**2
@@ -460,13 +465,13 @@ class IsaacMultiModalProcessor(BaseMultiModalProcessor):
465 assert isinstance(grid_thw, torch.Tensor)
466
467 feature_size = int(grid_thw.prod()) // merge_length
463 - repl_full = "<|image_pad|>" * feature_size
464 - return PromptUpdateDetails.select_text(repl_full, "<|image_pad|>")
468 + repl_full = image_pad_token_ids * feature_size
469 + return PromptUpdateDetails.select_token_ids(repl_full, image_pad_token_ids)
470
471 return [
472 PromptReplacement(
473 modality="image",
469 - target="<image>",
474 + target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
475 replacement=get_replacement_isaac,
476 )
477 ]
vllm/model_executor/models/kanana_v.py
+4 -1
@@ -33,6 +33,7 @@ from vllm.multimodal.processing import (
33 BaseProcessingInfo,
34 PromptReplacement,
35 PromptUpdate,
36 + cached_encode,
37 )
38 from vllm.sequence import IntermediateTensors
39 from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -541,6 +542,8 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
542 hf_processor_mm_kwargs: Mapping[str, object],
543 out_mm_kwargs: MultiModalKwargsItems,
544 ) -> Sequence[PromptUpdate]:
545 + tokenizer = self.info.get_tokenizer()
546 +
547 def get_replacement(idx: int) -> Sequence[int]:
548 out_item = out_mm_kwargs["image"][idx]
549 image_token_thw = out_item["image_token_thw"].data
@@ -552,7 +555,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo])
555 return [
556 PromptReplacement(
557 modality="image",
555 - target="<image>",
558 + target=cached_encode(tokenizer, "<image>", add_special_tokens=False),
559 replacement=get_replacement,
560 ),
561 ]
vllm/model_executor/models/lfm2_vl.py
+10 -5
@@ -42,6 +42,7 @@ from vllm.multimodal.processing import (
42 BaseProcessingInfo,
43 PromptReplacement,
44 PromptUpdateDetails,
45 + cached_encode,
46 )
47 from vllm.renderers import TokenizeParams
48 from vllm.sequence import IntermediateTensors
@@ -448,7 +449,8 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
449 out_mm_kwargs: MultiModalKwargsItems,
450 ) -> Sequence[PromptReplacement]:
451 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
451 - image_token = hf_processor.image_token
452 + image_token_id = hf_processor.image_token_id
453 + tokenizer = self.info.get_tokenizer()
454
455 def get_image_replacement_lfm2vl(item_idx: int):
456 images = mm_items.get_items("image", ImageProcessorItems)
@@ -463,15 +465,18 @@ class Lfm2VLMultiModalProcessor(BaseMultiModalProcessor[Lfm2VLProcessingInfo]):
465 processor=hf_processor,
466 mm_kwargs=hf_processor_mm_kwargs,
467 )
466 - return PromptUpdateDetails.select_text(
467 - image_repl,
468 - embed_text=image_token,
468 + image_repl_ids = cached_encode(
469 + tokenizer, image_repl, add_special_tokens=False
470 + )
471 + return PromptUpdateDetails.select_token_id(
472 + image_repl_ids,
473 + image_token_id,
474 )
475
476 return [
477 PromptReplacement(
478 modality="image",
474 - target=image_token,
479 + target=[image_token_id],
480 replacement=get_image_replacement_lfm2vl,
481 )
482 ]
vllm/model_executor/models/midashenglm.py
+1 -1
@@ -662,7 +662,7 @@ class MiDashengLMMultiModalProcessor(
662 return [
663 PromptReplacement(
664 modality="audio",
665 - target=audio_token,
665 + target=[audio_token_id],
666 replacement=get_replacement_midashenglm,
667 )
668 ]
vllm/model_executor/models/mimo_v2_omni.py
+1 -1
@@ -1112,7 +1112,7 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
1112 embed_t = torch.tensor(is_embed_mask)
1113 return PromptUpdateDetails(
1114 full=full,
1115 - is_embed=lambda _tok, _seq: embed_t,
1115 + is_embed=lambda _seq: embed_t,
1116 )
1117
1118 def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
vllm/model_executor/models/minicpmo.py
+15 -4
@@ -58,6 +58,7 @@ from vllm.multimodal.processing import (
58 PromptReplacement,
59 PromptUpdate,
60 PromptUpdateDetails,
61 + cached_encode,
62 )
63 from vllm.utils.tensor_schema import TensorSchema, TensorShape
64
@@ -502,7 +503,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
503 out_mm_kwargs=out_mm_kwargs,
504 )
505
505 - audio_placeholder = self.info.audio_pattern
506 + tokenizer = self.info.get_tokenizer()
507 + vocab = tokenizer.get_vocab()
508 +
509 + audio_placeholder = cached_encode(
510 + tokenizer, self.info.audio_pattern, add_special_tokens=False
511 + )
512 + unk_token_ids = [vocab["<unk>"]]
513
514 def get_audio_replacement(item_idx: int):
515 audios = mm_items.get_items(
@@ -517,9 +524,13 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
524 else:
525 audio_len = audios.get_audio_length(item_idx)
526
520 - return PromptUpdateDetails.select_text(
521 - self.get_audio_prompt_texts(audio_len),
522 - "<unk>",
527 + return PromptUpdateDetails.select_token_ids(
528 + cached_encode(
529 + tokenizer,
530 + self.get_audio_prompt_texts(audio_len),
531 + add_special_tokens=False,
532 + ),
533 + unk_token_ids,
534 )
535
536 return [
vllm/model_executor/models/minicpmv.py
+40 -29
@@ -88,7 +88,7 @@ from vllm.multimodal.processing.processor import (
88 PromptUpdate,
89 PromptUpdateDetails,
90 ResolvedPromptUpdate,
91 - _seq2text,
91 + cached_encode,
92 )
93 from vllm.platforms import current_platform
94 from vllm.sequence import IntermediateTensors
@@ -938,30 +938,24 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
938 mm_prompt_updates: MultiModalPromptUpdates,
939 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
940 """Apply multi-modal prompt updates to token IDs."""
941 - tokenizer = self.info.get_tokenizer()
942 -
941 new_token_ids, match_result = self._apply_token_matches(
942 token_ids,
943 mm_prompt_updates,
944 )
945
948 - # If the search text does not represent a special token,
949 - # it may have different token IDs in the prompt, because
950 - # the tokens may go across the boundaries of the search text.
951 - # ----
952 - # e.g. when searching for "foo" in "food", if "food" itself makes
953 - # up a token, then the token ID of "foo" will not appear at all
954 - # ----
955 - # Since it is inefficient to search for all possible tokenizations
956 - # of the search text in the prompt, we instead perform string-based
957 - # updates on the decoded token IDs, then encode them back.
946 + # If the target does not consist of special tokens, it may be
947 + # tokenized differently inside the prompt, so fall back to
948 + # performing the updates on the decoded text, then encoding the
949 + # result back. The segments are encoded separately to avoid BPE
950 + # merges across segment boundaries.
951 if not all(
952 all(update_idx is not None for update_idx in update_idxs)
953 for update_idxs in match_result.values()
954 ):
962 - new_token_ids, match_result = self._apply_text_matches_as_segmented_tokens(
963 - _seq2text(tokenizer, token_ids, use_cache=False),
955 + new_token_ids, match_result = self._apply_prompt_updates_via_text(
956 + token_ids,
957 mm_prompt_updates,
958 + encode_segments_separately=True,
959 )
960
961 matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list)
@@ -1062,6 +1056,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1056 additional_placeholders.append((modality, sub_pattern))
1057 placeholders += additional_placeholders
1058
1059 + vocab = tokenizer.get_vocab()
1060 + unk_token_ids = [vocab["<unk>"]]
1061 +
1062 def get_image_replacement(item_idx: int):
1063 images = mm_items.get_items(
1064 "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
@@ -1069,9 +1066,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1066
1067 image_size = images.get_image_size(item_idx)
1068
1072 - return PromptUpdateDetails.select_text(
1073 - self.get_image_prompt_texts(image_size, item_idx),
1074 - "<unk>",
1069 + return PromptUpdateDetails.select_token_ids(
1070 + cached_encode(
1071 + tokenizer,
1072 + self.get_image_prompt_texts(image_size, item_idx),
1073 + add_special_tokens=False,
1074 + ),
1075 + unk_token_ids,
1076 )
1077
1078 def get_video_replacement(item_idx: int):
@@ -1082,9 +1083,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1083 frame_size = videos.get_frame_size(item_idx)
1084 num_frames = videos.get_num_frames(item_idx)
1085
1085 - return PromptUpdateDetails.select_text(
1086 - self.get_video_prompt_texts(frame_size, num_frames),
1087 - "<unk>",
1086 + return PromptUpdateDetails.select_token_ids(
1087 + cached_encode(
1088 + tokenizer,
1089 + self.get_video_prompt_texts(frame_size, num_frames),
1090 + add_special_tokens=False,
1091 + ),
1092 + unk_token_ids,
1093 )
1094
1095 get_replacement = {
@@ -1094,7 +1099,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1099
1100 return [
1101 PromptReplacement(
1097 - modality=modality, target=pattern, replacement=get_replacement[modality]
1102 + modality=modality,
1103 + target=cached_encode(tokenizer, pattern, add_special_tokens=False),
1104 + replacement=get_replacement[modality],
1105 )
1106 for modality, pattern in placeholders
1107 ]
@@ -1114,7 +1121,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1121 image_processor = self.info.get_image_processor()
1122 version = self.info.get_model_version()
1123
1117 - text = _seq2text(tokenizer, cached_update.content.full)
1124 + text = tokenizer.decode(cached_update.content.full)
1125 prev_item_idx = cached_update.item_idx
1126
1127 if version == (2, 0) or version == (2, 5):
@@ -1130,13 +1137,17 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
1137
1138 embed_text = getattr(tokenizer, "image_token", "<unk>")
1139 new_update = new_update.with_content(
1133 - PromptUpdateDetails.select_text(
1134 - text.replace(
1135 - f"{im_start}{prev_item_idx}{im_end}",
1136 - f"{im_start}{new_item_idx}{im_end}",
1137 - 1,
1140 + PromptUpdateDetails.select_token_ids(
1141 + cached_encode(
1142 + tokenizer,
1143 + text.replace(
1144 + f"{im_start}{prev_item_idx}{im_end}",
1145 + f"{im_start}{new_item_idx}{im_end}",
1146 + 1,
1147 + ),
1148 + add_special_tokens=False,
1149 ),
1139 - embed_text,
1150 + cached_encode(tokenizer, embed_text, add_special_tokens=False),
1151 )
1152 )
1153
vllm/model_executor/models/minicpmv4_6.py
+48 -29
@@ -36,7 +36,7 @@ from vllm.multimodal.processing.processor import (
36 PromptReplacement,
37 PromptUpdateDetails,
38 ResolvedPromptUpdate,
39 - _seq2text,
39 + cached_encode,
40 )
41 from vllm.sequence import IntermediateTensors
42
@@ -338,6 +338,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
338 # than `<unk>`, so use those tokens as the embedding selector.
339 image_embed_text = getattr(tokenizer, "image_token", "<|image_pad|>")
340 video_embed_text = getattr(tokenizer, "video_token", "<|video_pad|>")
341 + vocab = tokenizer.get_vocab()
342 + image_embed_ids = [vocab[image_embed_text]]
343 + video_embed_ids = [vocab[video_embed_text]]
344
345 def get_image_replacement(item_idx: int):
346 images = mm_items.get_items(
@@ -345,13 +348,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
348 (MiniCPMVImageEmbeddingItems, ImageProcessorItems),
349 )
350 image_size = images.get_image_size(item_idx)
348 - return PromptUpdateDetails.select_text(
349 - self.get_image_prompt_texts(
350 - image_size,
351 - item_idx,
352 - downsample_mode=ds_mode,
351 + return PromptUpdateDetails.select_token_ids(
352 + cached_encode(
353 + tokenizer,
354 + self.get_image_prompt_texts(
355 + image_size,
356 + item_idx,
357 + downsample_mode=ds_mode,
358 + ),
359 + add_special_tokens=False,
360 ),
354 - image_embed_text,
361 + image_embed_ids,
362 )
363
364 def get_video_replacement(item_idx: int):
@@ -370,14 +377,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
377 width=int(image_sizes[0, 0].item()),
378 height=int(image_sizes[0, 1].item()),
379 )
373 - return PromptUpdateDetails.select_text(
374 - self.get_video_prompt_texts(
375 - frame_size,
376 - num_frames,
377 - downsample_mode=ds_mode,
378 - video_idx=item_idx,
380 + return PromptUpdateDetails.select_token_ids(
381 + cached_encode(
382 + tokenizer,
383 + self.get_video_prompt_texts(
384 + frame_size,
385 + num_frames,
386 + downsample_mode=ds_mode,
387 + video_idx=item_idx,
388 + ),
389 + add_special_tokens=False,
390 ),
380 - video_embed_text,
391 + video_embed_ids,
392 )
393
394 videos = mm_items.get_items(
@@ -386,14 +397,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
397 )
398 frame_size = videos.get_frame_size(item_idx)
399 num_frames = videos.get_num_frames(item_idx)
389 - return PromptUpdateDetails.select_text(
390 - self.get_video_prompt_texts(
391 - frame_size,
392 - num_frames,
393 - downsample_mode=ds_mode,
394 - video_idx=item_idx,
400 + return PromptUpdateDetails.select_token_ids(
401 + cached_encode(
402 + tokenizer,
403 + self.get_video_prompt_texts(
404 + frame_size,
405 + num_frames,
406 + downsample_mode=ds_mode,
407 + video_idx=item_idx,
408 + ),
409 + add_special_tokens=False,
410 ),
396 - video_embed_text,
411 + video_embed_ids,
412 )
413
414 get_replacement = {
@@ -404,7 +419,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
419 return [
420 PromptReplacement(
421 modality=modality,
407 - target=pattern,
422 + target=cached_encode(tokenizer, pattern, add_special_tokens=False),
423 replacement=get_replacement[modality],
424 )
425 for modality, pattern in placeholders
@@ -424,17 +439,21 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
439 id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
440 video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
441
427 - text = _seq2text(tokenizer, cached_update.content.full)
442 + text = tokenizer.decode(cached_update.content.full)
443 prev_item_idx = cached_update.item_idx
444
445 new_update = new_update.with_content(
431 - PromptUpdateDetails.select_text(
432 - text.replace(
433 - f"{id_start}{prev_item_idx}{id_end}",
434 - f"{id_start}{new_item_idx}{id_end}",
435 - 1,
446 + PromptUpdateDetails.select_token_ids(
447 + cached_encode(
448 + tokenizer,
449 + text.replace(
450 + f"{id_start}{prev_item_idx}{id_end}",
451 + f"{id_start}{new_item_idx}{id_end}",
452 + 1,
453 + ),
454 + add_special_tokens=False,
455 ),
437 - video_token,
456 + cached_encode(tokenizer, video_token, add_special_tokens=False),
457 )
458 )
459 return new_update
vllm/model_executor/models/mllama4.py
+9 -3
@@ -70,6 +70,7 @@ from vllm.multimodal.processing import (
70 PromptReplacement,
71 PromptUpdate,
72 PromptUpdateDetails,
73 + cached_encode,
74 )
75 from vllm.sequence import IntermediateTensors
76 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -666,9 +667,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
667
668 num_patches_per_chunk = self.info.get_patch_per_chunk(vision_config)
669 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
669 - image_token = hf_processor.image_token
670 img_patch_token = hf_processor.img_patch_token
671
672 + tokenizer = self.info.get_tokenizer()
673 + img_patch_token_ids = cached_encode(
674 + tokenizer, img_patch_token, add_special_tokens=False
675 + )
676 +
677 def get_replacement(item_idx: int):
678 out_item = out_mm_kwargs["image"][item_idx]
679 aspect_ratio = out_item["aspect_ratios"].data
@@ -678,12 +683,13 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
683 num_patches_per_chunk=num_patches_per_chunk,
684 )
685
681 - return PromptUpdateDetails.select_text(repl, img_patch_token)
686 + repl_ids = cached_encode(tokenizer, repl, add_special_tokens=False)
687 + return PromptUpdateDetails.select_token_ids(repl_ids, img_patch_token_ids)
688
689 return [
690 PromptReplacement(
691 modality="image",
686 - target=image_token,
692 + target=[hf_processor.image_token_id],
693 replacement=get_replacement,
694 )
695 ]
vllm/model_executor/models/molmo.py
+4 -1
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
61 PromptInsertion,
62 PromptUpdate,
63 PromptUpdateDetails,
64 + cached_encode,
65 )
66 from vllm.sequence import IntermediateTensors
67 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -1263,7 +1264,9 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1264 return [
1265 PromptInsertion(
1266 modality="image",
1266 - target=PromptIndexTargets.prefix("<|endoftext|>"),
1267 + target=PromptIndexTargets.prefix(
1268 + cached_encode(tokenizer, "<|endoftext|>", add_special_tokens=False),
1269 + ),
1270 insertion=get_insertion_molmo,
1271 )
1272 ]
vllm/model_executor/models/moss_audio.py
+2 -2
@@ -1393,7 +1393,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
1393 def get_replacement(
1394 item_idx: int,
1395 suffix_token_ids: list[int] | None = None,
1396 - ) -> PromptUpdateDetails[list[int]]:
1396 + ) -> PromptUpdateDetails:
1397 num_tokens = audio_token_lens[item_idx]
1398 if num_tokens == 0:
1399 raise ValueError("The audio is too short to be represented.")
@@ -1410,7 +1410,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
1410 processor.audio_end_id,
1411 *suffix_token_ids,
1412 ],
1413 - is_embed=lambda _tokenizer, _seq: torch.cat(
1413 + is_embed=lambda _seq: torch.cat(
1414 [
1415 torch.tensor([False]),
1416 is_embed,
vllm/model_executor/models/moss_transcribe_diarize.py
+5 -2
@@ -64,6 +64,7 @@ from vllm.multimodal.processing import (
64 PromptReplacement,
65 PromptUpdate,
66 PromptUpdateDetails,
67 + cached_encode,
68 )
69 from vllm.multimodal.processing.processor import ProcessorInputs
70 from vllm.sequence import IntermediateTensors
@@ -535,7 +536,7 @@ class MossTranscribeDiarizeMultiModalProcessor(
536 raise ValueError("Audio input is too short to produce any tokens.")
537 return num_tokens
538
538 - def get_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
539 + def get_replacement(item_idx: int) -> PromptUpdateDetails:
540 num_tokens = get_num_tokens(item_idx)
541 audio_tokens = processor._audio_span_ids(num_tokens)
542 return PromptUpdateDetails.select_token_id(
@@ -546,7 +547,9 @@ class MossTranscribeDiarizeMultiModalProcessor(
547 return [
548 PromptReplacement(
549 modality="audio",
549 - target=AUDIO_PLACEHOLDER,
550 + target=cached_encode(
551 + tokenizer, AUDIO_PLACEHOLDER, add_special_tokens=False
552 + ),
553 replacement=get_replacement,
554 ),
555 ]
vllm/model_executor/models/muse_glimmer.py
+2 -2
@@ -407,7 +407,7 @@ class MuseGlimmerMultiModalProcessor(
407 video_end_id = vocab["<|vid_end|>"]
408 video_separator_id = vocab["<|vid_frame_separator|>"]
409
410 - def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
410 + def image_replacement(item_idx: int) -> PromptUpdateDetails:
411 out_item = out_mm_kwargs["image"][item_idx]
412 num_tokens = int(out_item["image_feature_sizes"].data)
413 replacement = (
@@ -417,7 +417,7 @@ class MuseGlimmerMultiModalProcessor(
417 replacement, config.image_token_id
418 )
419
420 - def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
420 + def video_replacement(item_idx: int) -> PromptUpdateDetails:
421 out_item = out_mm_kwargs["video"][item_idx]
422 pixel_values = out_item["video_pixel_values"].data
423 num_groups = len(pixel_values)
vllm/model_executor/models/nano_nemotron_vl.py
+11 -3
@@ -70,6 +70,7 @@ from vllm.multimodal.processing.processor import (
70 BaseProcessingInfo,
71 PromptReplacement,
72 PromptUpdate,
73 + cached_encode,
74 )
75 from vllm.multimodal.video_prune.evs import (
76 compute_retained_tokens_count,
@@ -429,6 +430,9 @@ class NanoNemotronVLMultiModalProcessor(
430 hf_processor: NanoNemotronVLProcessor,
431 out_mm_data: BatchedTensorInputs,
432 ):
433 + tokenizer = self.info.get_tokenizer()
434 + vocab = tokenizer.get_vocab()
435 +
436 if "image_num_patches" in out_mm_data:
437 image_num_patches = out_mm_data["image_num_patches"]
438 assert isinstance(image_num_patches, torch.Tensor)
@@ -470,7 +474,7 @@ class NanoNemotronVLMultiModalProcessor(
474
475 return PromptReplacement(
476 modality="image",
473 - target="<image>",
477 + target=[vocab["<image>"]],
478 replacement=get_image_replacement,
479 )
480
@@ -480,6 +484,8 @@ class NanoNemotronVLMultiModalProcessor(
484 hf_processor: NanoNemotronVLProcessor,
485 out_mm_data: BatchedTensorInputs,
486 ):
487 + tokenizer = self.info.get_tokenizer()
488 +
489 if "video_num_patches" in out_mm_data:
490 video_num_patches = out_mm_data["video_num_patches"]
491 assert isinstance(video_num_patches, torch.Tensor)
@@ -546,7 +552,7 @@ class NanoNemotronVLMultiModalProcessor(
552
553 return PromptReplacement(
554 modality="video",
549 - target="<video>",
555 + target=cached_encode(tokenizer, "<video>", add_special_tokens=False),
556 replacement=get_video_replacement,
557 )
558
@@ -556,13 +562,15 @@ class NanoNemotronVLMultiModalProcessor(
562 hf_processor: NanoNemotronVLProcessor,
563 out_mm_data: BatchedTensorInputs,
564 ):
565 + tokenizer = self.info.get_tokenizer()
566 +
567 def get_audio_replacement(item_idx: int):
568 audios = mm_items.get_items("audio", AudioProcessorItems)
569 return hf_processor.get_audio_repl(audios.get(item_idx))
570
571 return PromptReplacement(
572 modality="audio",
565 - target=AUDIO_CONTEXT,
573 + target=cached_encode(tokenizer, AUDIO_CONTEXT, add_special_tokens=False),
574 replacement=get_audio_replacement,
575 )
576
vllm/model_executor/models/nvlm_d.py
+10 -3
@@ -26,6 +26,7 @@ from vllm.multimodal.parse import (
26 from vllm.multimodal.processing import (
27 PromptReplacement,
28 PromptUpdateDetails,
29 + cached_encode,
30 )
31 from vllm.transformers_utils.processors.internvl import InternVLImageProcessor
32 from vllm.transformers_utils.processors.nvlm_d import NVLMProcessor
@@ -106,6 +107,8 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
107 hf_processor: NVLMProcessor,
108 out_mm_data: BatchedTensorInputs,
109 ):
110 + tokenizer = self.info.get_tokenizer()
111 +
112 if "image_num_patches" in out_mm_data:
113 image_num_patches = out_mm_data["image_num_patches"]
114 assert isinstance(image_num_patches, torch.Tensor)
@@ -138,14 +141,18 @@ class NVLMMultiModalProcessor(BaseInternVLMultiModalProcessor[NVLMProcessingInfo
141
142 repl = hf_processor.get_image_repl(num_patches, num_features=feature_size)
143
141 - return PromptUpdateDetails.select_text(
142 - repl.full + "\n", hf_processor.ctx_image_token
144 + # `repl.full` ends in the special `</Image>` token, so there is
145 + # no BPE merge across the boundary with the newline
146 + newline_ids = cached_encode(tokenizer, "\n", add_special_tokens=False)
147 +
148 + return PromptUpdateDetails.select_token_id(
149 + repl.full + newline_ids, hf_processor.ctx_image_token_id
150 )
151
152 # See note in dummy data regarding why we have the extra newline
153 return PromptReplacement(
154 modality="image",
148 - target="<image>\n",
155 + target=cached_encode(tokenizer, "<image>\n", add_special_tokens=False),
156 replacement=get_replacement_nvlm,
157 )
158
vllm/model_executor/models/openvla.py
+1 -1
@@ -339,7 +339,7 @@ class OpenVLAMultiModalProcessor(BaseMultiModalProcessor[OpenVLAProcessingInfo])
339 tokenizer = self.info.get_tokenizer()
340 bos_token_id = tokenizer.bos_token_id
341
342 - def get_insertion(item_idx: int) -> PromptUpdateDetails[list[int]]:
342 + def get_insertion(item_idx: int) -> PromptUpdateDetails:
343 images = mm_items.get_items(
344 "image", (ImageEmbeddingItems, ImageProcessorItems)
345 )
vllm/model_executor/models/ovis.py
+4 -1
@@ -52,6 +52,7 @@ from vllm.multimodal.processing import (
52 BaseMultiModalProcessor,
53 BaseProcessingInfo,
54 PromptReplacement,
55 + cached_encode,
56 )
57 from vllm.renderers import TokenizeParams
58 from vllm.sequence import IntermediateTensors
@@ -383,6 +384,8 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
384 hf_processor_mm_kwargs: Mapping[str, object],
385 out_mm_kwargs: MultiModalKwargsItems,
386 ) -> list[PromptReplacement]:
387 + tokenizer = self.info.get_tokenizer()
388 +
389 def get_replacement_ovis(item_idx: int):
390 out_item = out_mm_kwargs["image"][item_idx]
391 grid = out_item["grids"].data
@@ -393,7 +396,7 @@ class OvisMultiModalProcessor(BaseMultiModalProcessor[OvisProcessingInfo]):
396 return [
397 PromptReplacement(
398 modality="image",
396 - target=IMAGE_TOKEN,
399 + target=cached_encode(tokenizer, IMAGE_TOKEN, add_special_tokens=False),
400 replacement=get_replacement_ovis,
401 ),
402 ]
vllm/model_executor/models/phi3v.py
+14 -2
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
54 PromptReplacement,
55 PromptUpdate,
56 ResolvedPromptUpdate,
57 + cached_encode,
58 )
59 from vllm.sequence import IntermediateTensors
60 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -436,8 +437,14 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
437 out_mm_kwargs: MultiModalKwargsItems,
438 ) -> Sequence[PromptUpdate]:
439 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
440 + tokenizer = self.info.get_tokenizer()
441 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
442
443 + def get_image_token_ids(item_idx: int) -> list[int]:
444 + return cached_encode(
445 + tokenizer, image_tokens[item_idx], add_special_tokens=False
446 + )
447 +
448 def get_replacement_phi3v(item_idx: int):
449 images = mm_items.get_items(
450 "image", (ImageEmbeddingItems, ImageProcessorItems)
@@ -458,7 +465,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
465 return [
466 PromptReplacement(
467 modality="image",
461 - target=image_tokens.__getitem__,
468 + target=get_image_token_ids,
469 replacement=get_replacement_phi3v,
470 )
471 ]
@@ -475,8 +482,13 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
482
483 if cached_update.modality == "image":
484 hf_processor = self.info.get_hf_processor()
485 + tokenizer = self.info.get_tokenizer()
486 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
479 - new_update = new_update.with_target(image_tokens[new_item_idx])
487 + new_update = new_update.with_target(
488 + cached_encode(
489 + tokenizer, image_tokens[new_item_idx], add_special_tokens=False
490 + )
491 + )
492
493 return new_update
494
vllm/model_executor/models/phi4mm.py
+27 -4
@@ -47,6 +47,7 @@ from vllm.multimodal.processing.processor import (
47 PromptReplacement,
48 PromptUpdate,
49 ResolvedPromptUpdate,
50 + cached_encode,
51 )
52 from vllm.sequence import IntermediateTensors
53 from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -934,8 +935,20 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
935 hf_processor_mm_kwargs: Mapping[str, Any],
936 out_mm_kwargs: MultiModalKwargsItems,
937 ) -> Sequence[PromptUpdate]:
938 + tokenizer = self.info.get_tokenizer()
939 image_tokens: list[str] = self.info.image_tokens # type: ignore
940 audio_tokens: list[str] = self.info.audio_tokens # type: ignore
941 +
942 + def get_image_token_ids(item_idx: int) -> list[int]:
943 + return cached_encode(
944 + tokenizer, image_tokens[item_idx], add_special_tokens=False
945 + )
946 +
947 + def get_audio_token_ids(item_idx: int) -> list[int]:
948 + return cached_encode(
949 + tokenizer, audio_tokens[item_idx], add_special_tokens=False
950 + )
951 +
952 feature_extractor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
953 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
954
@@ -970,12 +983,12 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
983 return [
984 PromptReplacement(
985 modality="image",
973 - target=image_tokens.__getitem__,
986 + target=get_image_token_ids,
987 replacement=get_image_replacement_phi4mm,
988 ),
989 PromptReplacement(
990 modality="audio",
978 - target=audio_tokens.__getitem__,
991 + target=get_audio_token_ids,
992 replacement=get_audio_replacement_phi4mm,
993 ),
994 ]
@@ -990,12 +1003,22 @@ class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
1003 new_item_idx,
1004 )
1005
1006 + tokenizer = self.info.get_tokenizer()
1007 +
1008 if cached_update.modality == "image":
1009 image_tokens: list[str] = self.info.image_tokens # type: ignore
995 - new_update = new_update.with_target(image_tokens[new_item_idx])
1010 + new_update = new_update.with_target(
1011 + cached_encode(
1012 + tokenizer, image_tokens[new_item_idx], add_special_tokens=False
1013 + )
1014 + )
1015 elif cached_update.modality == "audio":
1016 audio_tokens: list[str] = self.info.audio_tokens # type: ignore
998 - new_update = new_update.with_target(audio_tokens[new_item_idx])
1017 + new_update = new_update.with_target(
1018 + cached_encode(
1019 + tokenizer, audio_tokens[new_item_idx], add_special_tokens=False
1020 + )
1021 + )
1022
1023 return new_update
1024
vllm/model_executor/models/phi4siglip.py
+6 -1
@@ -34,6 +34,7 @@ from vllm.multimodal.processing import (
34 from vllm.multimodal.processing.processor import (
35 BaseMultiModalProcessor,
36 BaseProcessingInfo,
37 + cached_encode,
38 )
39 from vllm.sequence import IntermediateTensors
40 from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -193,6 +194,8 @@ class Phi4SiglipMultiModalProcessor(
194 hf_processor_mm_kwargs: Mapping[str, Any],
195 out_mm_kwargs: MultiModalKwargsItems,
196 ) -> Sequence[PromptUpdate]:
197 + tokenizer = self.info.get_tokenizer()
198 +
199 def get_replacement(item_idx: int):
200 # Read the actual patch grid from the NaFlex processor's
201 # spatial_shapes output (same pattern as LFM2-VL). This avoids
@@ -207,7 +210,9 @@ class Phi4SiglipMultiModalProcessor(
210 return [
211 PromptReplacement(
212 modality="image",
210 - target=DEFAULT_IMAGE_TOKEN,
213 + target=cached_encode(
214 + tokenizer, DEFAULT_IMAGE_TOKEN, add_special_tokens=False
215 + ),
216 replacement=get_replacement,
217 ),
218 ]
vllm/model_executor/models/pixtral.py
+1 -1
@@ -301,7 +301,7 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
301 return [
302 PromptReplacement(
303 modality="image",
304 - target="", # Never match the prompt (see below note)
304 + target=[], # Never match the prompt (see below note)
305 replacement=get_replacement,
306 ),
307 ]
vllm/model_executor/models/qwen2_5_omni_thinker.py
+3 -3
@@ -779,17 +779,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
779 return [
780 PromptReplacement(
781 modality="audio",
782 - target=audio_token,
782 + target=[audio_token_id],
783 replacement=get_replacement_qwen2_audio,
784 ),
785 PromptReplacement(
786 modality="image",
787 - target=image_token,
787 + target=[image_token_id],
788 replacement=partial(get_replacement_qwen2_vision, modality="image"),
789 ),
790 PromptReplacement(
791 modality="video",
792 - target=video_token,
792 + target=[video_token_id],
793 replacement=video_replacement_fn,
794 ),
795 ]
vllm/model_executor/models/qwen3_asr.py
+1 -1
@@ -328,7 +328,7 @@ class Qwen3ASRMultiModalProcessor(
328 return [
329 PromptReplacement(
330 modality="audio",
331 - target=audio_token,
331 + target=[audio_token_id],
332 replacement=get_replacement_qwen2_audio,
333 ),
334 ]
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+3 -3
@@ -1493,17 +1493,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1493 return [
1494 PromptReplacement(
1495 modality="audio",
1496 - target=audio_token,
1496 + target=[audio_token_id],
1497 replacement=get_replacement_qwen2_audio,
1498 ),
1499 PromptReplacement(
1500 modality="image",
1501 - target=image_token,
1501 + target=[image_token_id],
1502 replacement=partial(get_replacement_qwen2_vision, modality="image"),
1503 ),
1504 PromptReplacement(
1505 modality="video",
1506 - target=video_token,
1506 + target=[video_token_id],
1507 replacement=video_replacement_fn,
1508 ),
1509 ]
vllm/model_executor/models/qwen3_vl.py
+8 -7
@@ -1555,17 +1555,19 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1555
1556 if self._expands_only_video_token(hf_processor):
1557 # transformers>=5.10 expands only the bare video_token
1558 - video_target = hf_processor.video_token
1558 + video_target = [video_token_id]
1559 else:
1560 # Old-style processors expand the full placeholder
1561 - # NOTE: We match string on purpose since searching sequence of
1562 - # token ids takes more time.
1563 - video_target = "<|vision_start|><|video_pad|><|vision_end|>"
1561 + video_target = [
1562 + vision_start_token_id,
1563 + video_token_id,
1564 + vision_end_token_id,
1565 + ]
1566
1567 return [
1568 PromptReplacement(
1569 modality="image",
1568 - target=hf_processor.image_token,
1570 + target=[hf_processor.image_token_id],
1571 replacement=get_image_replacement_qwen3vl,
1572 ),
1573 PromptReplacement(
@@ -1585,7 +1587,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1587 vision_end_token_id: int,
1588 video_token_id: int,
1589 select_token_id: bool = False,
1588 - ) -> PromptUpdateDetails[list[int]]:
1590 + ) -> PromptUpdateDetails:
1591 """Build prompt replacement for a video in Qwen3VL format.
1592
1593 The replacement structure for each frame is:
@@ -1609,7 +1611,6 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1611
1612 # Tokenize timestamp strings independently to avoid tokenizer merging
1613 # tokens across boundaries.
1612 - # TODO: switch to `_seq2tokens` which has some caching.
1614 timestamp_token_ids = [
1615 tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
1616 for timestamp in timestamps
vllm/model_executor/models/transformers/multimodal.py
+12 -1
@@ -56,6 +56,7 @@ from vllm.multimodal.processing import (
56 PromptUpdate,
57 PromptUpdateDetails,
58 TimingContext,
59 + cached_encode,
60 )
61 from vllm.sequence import IntermediateTensors
62 from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -700,6 +701,15 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
701 """Replace each modality's placeholder token with the token ids that item's
702 replacement text encodes to, marking which of them hold embeddings."""
703 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
704 + tokenizer = self.info.get_tokenizer()
705 +
706 + def get_target_token_ids(token: str | int | Sequence[int]) -> list[int]:
707 + if isinstance(token, str):
708 + return cached_encode(tokenizer, token, add_special_tokens=False)
709 + if isinstance(token, int):
710 + return [token]
711 + return list(token)
712 +
713 updates = []
714 for modality, items in out_mm_kwargs.items():
715 # Popped so they are neither cached nor sent to the model; the updates
@@ -711,10 +721,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
721 )
722 for item in items
723 ]
724 + token = getattr(hf_processor, f"{modality}_token")
725 updates.append(
726 PromptReplacement(
727 modality=modality,
717 - target=getattr(hf_processor, f"{modality}_token"),
728 + target=get_target_token_ids(token),
729 replacement=replacements.__getitem__,
730 )
731 )
vllm/model_executor/models/ultravox.py
+11 -1
@@ -39,6 +39,7 @@ from vllm.multimodal.processing import (
39 BaseProcessingInfo,
40 PromptReplacement,
41 PromptUpdate,
42 + cached_encode,
43 )
44 from vllm.renderers import TokenizeParams
45 from vllm.sequence import IntermediateTensors
@@ -298,6 +299,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
299 out_mm_kwargs: MultiModalKwargsItems,
300 ) -> Sequence[PromptUpdate]:
301 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
302 + tokenizer = self.info.get_tokenizer()
303
304 replacement_id = hf_processor.audio_replacement_token_id # type: ignore
305
@@ -322,7 +324,15 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
324 return [
325 PromptReplacement(
326 modality="audio",
325 - target="<|audio|>",
327 + # `<|audio|>` is paired with `replacement_id` in
328 + # `UltravoxProcessingInfo.get_hf_processor`, but it is not
329 + # guaranteed to be a single vocab entry, so match the encoded
330 + # placeholder text instead of `replacement_id`
331 + target=cached_encode(
332 + tokenizer,
333 + hf_processor.audio_token_replacement, # type: ignore
334 + add_special_tokens=False,
335 + ),
336 replacement=get_replacement_ultravox,
337 )
338 ]
vllm/model_executor/models/voxtral.py
+1 -1
@@ -310,7 +310,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
310 return [
311 PromptReplacement(
312 modality="audio",
313 - target="", # Never match the prompt (see below note)
313 + target=[], # Never match the prompt (see below note)
314 replacement=get_replacement,
315 ),
316 ]
vllm/models/dots3_note/common/processor.py
+6 -6
@@ -722,7 +722,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
722 image_end_id = vocab[IMAGE_END]
723 merge_size = self.info.image_processor.merge_size # type: ignore[union-attr]
724
725 - def image_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
725 + def image_replacement(item_idx: int) -> PromptUpdateDetails:
726 grid = out_mm_kwargs["image"][item_idx]["image_grid_thw"].data
727 assert isinstance(grid, torch.Tensor)
728 num_tokens = int(grid.prod()) // merge_size**2
@@ -750,7 +750,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
750 * int(config.get("merge_factor", 1))
751 )
752
753 - def audio_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
753 + def audio_replacement(item_idx: int) -> PromptUpdateDetails:
754 length = out_mm_kwargs["audio"][item_idx]["audio_lengths"].data
755 assert isinstance(length, torch.Tensor)
756 num_tokens = math.ceil(int(length.item()) / stride)
@@ -768,7 +768,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
768 )
769 if "video" in out_mm_kwargs:
770
771 - def video_replacement(item_idx: int) -> PromptUpdateDetails[list[int]]:
771 + def video_replacement(item_idx: int) -> PromptUpdateDetails:
772 item = out_mm_kwargs["video"][item_idx]
773 input_ids_data = item["video_input_ids"].data
774 embed_mask_data = item["video_embed_mask"].data
@@ -777,8 +777,8 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
777 input_ids = input_ids_data.tolist()
778 embed_mask = embed_mask_data.bool()
779
780 - def select_video_embeds(tokenizer, full) -> torch.Tensor:
781 - del tokenizer, full
780 + def select_video_embeds(full) -> torch.Tensor:
781 + del full
782 return embed_mask
783
784 return PromptUpdateDetails(
@@ -789,7 +789,7 @@ class Dots3NoteMultiModalProcessor(BaseMultiModalProcessor[Dots3NoteProcessingIn
789 updates.append(
790 PromptReplacement(
791 modality="video",
792 - target=VIDEO_PLACEHOLDER,
792 + target=[vocab[VIDEO_PLACEHOLDER]],
793 replacement=video_replacement,
794 )
795 )
vllm/models/kimi_k3/common/mm_preprocess.py
+10 -3
@@ -25,6 +25,7 @@ from vllm.multimodal.processing import (
25 PromptReplacement,
26 PromptUpdate,
27 PromptUpdateDetails,
28 + cached_encode,
29 )
30 from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
31 from vllm.transformers_utils.processor import cached_get_image_processor
@@ -268,8 +269,9 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
269 media_token_id = self.info.media_token_id
270 media_token = self.info.media_token
271 image_placeholder = self.info.get_hf_config().image_placeholder
272 + tokenizer = self.info.get_tokenizer()
273
272 - def get_replacement(item_idx: int) -> PromptUpdateDetails[str]:
274 + def get_replacement(item_idx: int) -> PromptUpdateDetails:
275 images = mm_items.get_items("image", ImageProcessorItems)
276 image = images.get(item_idx)
277 if image is None:
@@ -296,12 +298,17 @@ class KimiK3MultiModalProcessor(BaseMultiModalProcessor[KimiK3ProcessingInfo]):
298 f"{pads}<|media_end|>"
299 )
300
299 - return PromptUpdateDetails.select_token_id(full, media_token_id)
301 + return PromptUpdateDetails.select_token_id(
302 + cached_encode(tokenizer, full, add_special_tokens=False),
303 + media_token_id,
304 + )
305
306 return [
307 PromptReplacement(
308 modality="image",
304 - target=image_placeholder,
309 + target=cached_encode(
310 + tokenizer, image_placeholder, add_special_tokens=False
311 + ),
312 replacement=get_replacement,
313 ),
314 ]
vllm/multimodal/processing/__init__.py
+2
@@ -11,6 +11,7 @@ from .processor import (
11 PromptReplacement,
12 PromptUpdate,
13 PromptUpdateDetails,
14 + cached_encode,
15 )
16
17 __all__ = [
@@ -20,6 +21,7 @@ __all__ = [
21 "BaseDummyInputsBuilder",
22 "ProcessorInputs",
23 "BaseMultiModalProcessor",
24 + "cached_encode",
25 "EncDecMultiModalProcessor",
26 "PromptUpdate",
27 "PromptIndexTargets",
vllm/multimodal/processing/processor.py
+222 -310
@@ -12,10 +12,8 @@ from typing import (
12 NamedTuple,
13 Protocol,
14 TypeAlias,
15 - cast,
15 )
16
18 -import regex as re
17 import torch
18 from typing_extensions import TypeVar, assert_never
19
@@ -53,74 +51,31 @@ else:
51
52 logger = init_logger(__name__)
53
56 -_S = TypeVar("_S", str, list[int])
57 -
58 -
59 -PromptSeq: TypeAlias = str | list[int]
60 -"""A token sequence (list of token IDs) or text."""
61 -
54
55 @lru_cache(maxsize=2048)
64 -def _cached_encode(
56 +def cached_encode(
57 tokenizer: TokenizerLike,
58 text: str,
59 *,
60 add_special_tokens: bool = True,
61 ) -> list[int]:
62 + """Encode text while caching repeated tokenizer calls."""
63 return tokenizer.encode(text, add_special_tokens=add_special_tokens)
64
65
73 -@lru_cache(maxsize=2048)
74 -def _cached_decode(
75 - tokenizer: TokenizerLike,
76 - token_ids: tuple[int, ...],
77 - *,
78 - skip_special_tokens: bool = False,
79 -) -> str:
80 - return tokenizer.decode(list(token_ids), skip_special_tokens=skip_special_tokens)
81 -
82 -
83 -def _seq2text(
84 - tokenizer: TokenizerLike | None,
85 - seq: PromptSeq,
86 - *,
87 - use_cache: bool = True,
88 -) -> str:
89 - if isinstance(seq, str):
90 - return seq
91 -
92 - if tokenizer is None:
93 - raise ValueError("You cannot decode tokens when `skip_tokenizer_init=True`")
94 -
95 - if not use_cache:
96 - return tokenizer.decode(seq)
97 -
98 - return _cached_decode(tokenizer, tuple(seq))
99 -
100 -
101 -def _seq2tokens(
102 - tokenizer: TokenizerLike | None,
103 - seq: PromptSeq,
104 - *,
105 - use_cache: bool = True,
106 -) -> list[int]:
107 - if isinstance(seq, str):
108 - if tokenizer is None:
109 - raise ValueError("You cannot encode text when `skip_tokenizer_init=True`")
110 -
111 - if not use_cache:
112 - return tokenizer.encode(seq, add_special_tokens=False)
113 -
114 - return _cached_encode(tokenizer, seq, add_special_tokens=False)
115 -
116 - return seq
66 +class _GetMatchIndex(Protocol):
67 + def __call__(
68 + self,
69 + prompt: list[int],
70 + start_idx: int = 0,
71 + ) -> int | None: ...
72
73
119 -class _GetMatchIndex(Protocol):
74 +class _GetTextMatchIndex(Protocol):
75 def __call__(
76 self,
122 - tokenizer: TokenizerLike | None,
123 - prompt: PromptSeq,
77 + tokenizer: TokenizerLike,
78 + prompt: str,
79 start_idx: int = 0,
80 ) -> int | None: ...
81
@@ -131,6 +86,14 @@ class PromptIndex:
86
87 get_match_index: _GetMatchIndex
88
89 + get_text_match_index: _GetTextMatchIndex
90 + """
91 + Resolves the index against a decoded text prompt.
92 +
93 + Used only by the private text-based fallback of
94 + [`BaseMultiModalProcessor._apply_prompt_updates`][vllm.multimodal.processing.BaseMultiModalProcessor].
95 + """
96 +
97
98 class PromptIndexTargets:
99 @staticmethod
@@ -140,35 +103,40 @@ class PromptIndexTargets:
103
104 This results in a match even if the prompt is empty.
105 """
143 - return PromptIndex(lambda tokenizer, prompt, start_idx=0: 0)
106 + return PromptIndex(
107 + lambda prompt, start_idx=0: 0,
108 + lambda tokenizer, prompt, start_idx=0: 0,
109 + )
110
111 @staticmethod
146 - def prefix(seq: PromptSeq) -> PromptIndex:
112 + def prefix(seq: list[int]) -> PromptIndex:
113 """
114 Resolves to a location in the prompt after the given prefix.
115 """
116
117 def get_match_index(
152 - tokenizer: TokenizerLike | None,
153 - prompt: PromptSeq,
118 + prompt: list[int],
119 start_idx: int = 0,
120 ) -> int | None:
121 if start_idx != 0:
122 return None
123
159 - prefix = seq
124 + match_idx = len(seq)
125 + return match_idx if prompt[:match_idx] == seq else None
126
161 - if isinstance(prompt, str):
162 - # Make both `str`
163 - prefix = _seq2text(tokenizer, prefix, use_cache=False)
164 - else:
165 - # Make both `list[int]`
166 - prefix = _seq2tokens(tokenizer, prefix, use_cache=False)
127 + def get_text_match_index(
128 + tokenizer: TokenizerLike,
129 + prompt: str,
130 + start_idx: int = 0,
131 + ) -> int | None:
132 + if start_idx != 0:
133 + return None
134
135 + prefix = tokenizer.decode(seq)
136 match_idx = len(prefix)
137 return match_idx if prompt[:match_idx] == prefix else None
138
171 - return PromptIndex(get_match_index)
139 + return PromptIndex(get_match_index, get_text_match_index)
140
141 @staticmethod
142 def end() -> PromptIndex:
@@ -177,33 +145,36 @@ class PromptIndexTargets:
145
146 This results in a match even if the prompt is empty.
147 """
180 - return PromptIndex(lambda tokenizer, prompt, start_idx=0: len(prompt))
148 + return PromptIndex(
149 + lambda prompt, start_idx=0: len(prompt),
150 + lambda tokenizer, prompt, start_idx=0: len(prompt),
151 + )
152
153
183 -UpdateTarget: TypeAlias = PromptSeq | PromptIndex
154 +UpdateTarget: TypeAlias = list[int] | PromptIndex
155 """
185 -The token sequence or text to update.
156 +The token sequence to update.
157 """
158
159 PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget
160 """
161 Given the index of the processed item within
162 [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
192 -output the corresponding token sequence (or text).
163 +output the corresponding token sequence.
164
194 -For convenience, you can directly pass in the token sequence (or text)
165 +For convenience, you can directly pass in the token sequence
166 instead of a function if it does not depend on the input.
167 """
168
169
170 @dataclass
200 -class PromptUpdateDetails(Generic[_S]):
201 - """Details about the token sequence or text that are part of the update."""
171 +class PromptUpdateDetails:
172 + """Details about the token sequence that is part of the update."""
173
203 - full: _S
174 + full: list[int]
175 """The full content."""
176
206 - is_embed: Callable[[TokenizerLike | None, PromptSeq], torch.Tensor] | None = None
177 + is_embed: Callable[[list[int]], torch.Tensor] | None = None
178 """
179 Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
180 return a boolean mask of shape `(len(full),)` indicating which positions
@@ -216,56 +187,36 @@ class PromptUpdateDetails(Generic[_S]):
187 """
188
189 @staticmethod
219 - def from_seq(seq: _S) -> "PromptUpdateDetails[_S]":
190 + def from_seq(seq: list[int]) -> "PromptUpdateDetails":
191 return PromptUpdateDetails(full=seq)
192
222 - @staticmethod
223 - def select_text(
224 - seq: _S,
225 - embed_text: str,
226 - ) -> "PromptUpdateDetails[_S]":
227 - def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
228 - embed_token_ids = _seq2tokens(tokenizer, embed_text, use_cache=False)
229 - token_ids = _seq2tokens(tokenizer, full)
230 -
231 - return torch.isin(
232 - torch.tensor(token_ids),
233 - torch.tensor(embed_token_ids),
234 - )
235 -
236 - return PromptUpdateDetails(full=seq, is_embed=is_embed)
237 -
193 @staticmethod
194 def select_token_id(
240 - seq: _S,
195 + seq: list[int],
196 embed_token_id: int,
242 - ) -> "PromptUpdateDetails[_S]":
243 - def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
244 - token_ids = _seq2tokens(tokenizer, full)
245 -
246 - return torch.tensor(token_ids) == embed_token_id
197 + ) -> "PromptUpdateDetails":
198 + def is_embed(full: list[int]) -> torch.Tensor:
199 + return torch.tensor(full) == embed_token_id
200
201 return PromptUpdateDetails(full=seq, is_embed=is_embed)
202
203 @staticmethod
204 def select_token_ids(
252 - seq: _S,
205 + seq: list[int],
206 embed_token_ids: list[int],
254 - ) -> "PromptUpdateDetails[_S]":
255 - def is_embed(tokenizer: TokenizerLike | None, full: PromptSeq) -> torch.Tensor:
256 - token_ids = _seq2tokens(tokenizer, full)
257 -
207 + ) -> "PromptUpdateDetails":
208 + def is_embed(full: list[int]) -> torch.Tensor:
209 return torch.isin(
259 - torch.tensor(token_ids),
210 + torch.tensor(full),
211 torch.tensor(embed_token_ids),
212 )
213
214 return PromptUpdateDetails(full=seq, is_embed=is_embed)
215
216
266 -PromptUpdateInfo: TypeAlias = PromptSeq | PromptUpdateDetails
217 +PromptUpdateInfo: TypeAlias = list[int] | PromptUpdateDetails
218 """
268 -The token sequence or text that are part of the update.
219 +The token sequence that is part of the update.
220
221 If only part of the content corresponds to feature placeholders, you can
222 use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
@@ -276,9 +227,9 @@ PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdat
227 """
228 Given the index of the processed item within
229 [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
279 -output the corresponding token sequence (or text).
230 +output the corresponding token sequence.
231
281 -For convenience, you can directly pass in the token sequence (or text)
232 +For convenience, you can directly pass in the token sequence
233 instead of a function if it does not depend on the input.
234 """
235
@@ -292,13 +243,20 @@ class UpdateMode(str, Enum):
243 class PromptUpdate(ABC):
244 """
245 Defines how to update a prompt with placeholder tokens.
246 +
247 + Note:
248 + The target and content are token sequences. When converting text
249 + to token sequences, remember to encode it with
250 + `add_special_tokens=False` to match how the prompt itself is
251 + tokenized. Otherwise, the updated prompt may contain duplicated
252 + special tokens or fail to match the target.
253 """
254
255 modality: str
256 """The modality for which the update is made."""
257
258 target: PromptUpdateTarget
301 - """The token sequence (or text) to update."""
259 + """The token sequence to update."""
260
261 @property
262 @abstractmethod
@@ -357,8 +315,8 @@ class PromptInsertion(PromptUpdate):
315 ```python
316 PromptInsertion(
317 modality="image",
360 - target="<s>",
361 - insertion="<image>" * image_feature_size,
318 + target=[bos_token_id],
319 + insertion=[image_token_id] * image_feature_size,
320 )
321 ```
322
@@ -368,7 +326,7 @@ class PromptInsertion(PromptUpdate):
326 PromptInsertion(
327 modality="image",
328 target=PromptIndexTargets.start(),
371 - insertion="<image>" * image_feature_size,
329 + insertion=[image_token_id] * image_feature_size,
330 )
331 ```
332
@@ -377,8 +335,8 @@ class PromptInsertion(PromptUpdate):
335 ```python
336 PromptInsertion(
337 modality="image",
380 - target=PromptIndexTargets.prefix("Images:"),
381 - insertion="<image>" * image_feature_size,
338 + target=PromptIndexTargets.prefix(images_prefix_token_ids),
339 + insertion=[image_token_id] * image_feature_size,
340 )
341 ```
342
@@ -388,7 +346,7 @@ class PromptInsertion(PromptUpdate):
346 PromptInsertion(
347 modality="image",
348 target=PromptIndexTargets.end(),
391 - insertion="<image>" * image_feature_size,
349 + insertion=[image_token_id] * image_feature_size,
350 )
351 ```
352 """
@@ -397,10 +355,10 @@ class PromptInsertion(PromptUpdate):
355 """
356 Given the index of the processed item within
357 [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
400 - output the token sequence (or text) to insert right after
358 + output the token sequence to insert right after
359 [`target`][vllm.multimodal.processing.PromptUpdate.target].
360
403 - For convenience, you can directly pass in the token sequence (or text)
361 + For convenience, you can directly pass in the token sequence
362 instead of a function if it does not depend on the input.
363 """
364
@@ -427,8 +385,8 @@ class PromptReplacement(PromptUpdate):
385 ```python
386 PromptReplacement(
387 modality="image",
430 - target="<image>",
431 - replacement="<image>" * image_feature_size,
388 + target=[image_token_id],
389 + replacement=[image_token_id] * image_feature_size,
390 )
391 ```
392
@@ -436,26 +394,6 @@ class PromptReplacement(PromptUpdate):
394 and `<image_eos>`, which are not supposed to be passed to the vision
395 encoder:
396
439 - ```python
440 - PromptReplacement(
441 - modality="image",
442 - target="<image>",
443 - replacement=PromptUpdateDetails(
444 - full="".join(
445 - [
446 - "<image_bos>",
447 - "<image>" * image_feature_size,
448 - "<image_eos>",
449 - ]
450 - ),
451 - features="<image>" * image_feature_size,
452 - ),
453 - )
454 - ```
455 -
456 - To avoid unnecessary tokenization during prompt replacement,
457 - we recommended passing token sequences instead of text:
458 -
397 ```python
398 PromptReplacement(
399 modality="image",
@@ -464,7 +402,7 @@ class PromptReplacement(PromptUpdate):
402 full=(
403 [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]
404 ),
467 - features=[image_token_id] * image_feature_size,
405 + is_embed=lambda full: torch.tensor(full) == image_token_id,
406 ),
407 )
408 ```
@@ -474,10 +412,10 @@ class PromptReplacement(PromptUpdate):
412 """
413 Given the index of the processed item within
414 [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
477 - output the token sequence (or text) to replace
415 + output the token sequence to replace
416 [`target`][vllm.multimodal.processing.PromptUpdate.target].
417
480 - For convenience, you can directly pass in the token sequence (or text)
418 + For convenience, you can directly pass in the token sequence
419 instead of a function if it does not depend on the input.
420 """
421
@@ -520,7 +458,7 @@ class PromptTargetMatch(NamedTuple):
458 class ResolvedPromptUpdate:
459 """
460 A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its
523 - lazy attributes resolved, apart from those related to tokenization.
461 + lazy attributes resolved.
462 """
463
464 modality: str
@@ -533,7 +471,7 @@ class ResolvedPromptUpdate:
471 """Defines how to update the prompt."""
472
473 target: UpdateTarget
536 - """The token sequence (or text) to update."""
474 + """The token sequence to update."""
475
476 content: PromptUpdateDetails = field(repr=False)
477 """The placeholder tokens that are part of the update."""
@@ -541,7 +479,6 @@ class ResolvedPromptUpdate:
479 def iter_token_matches(
480 self,
481 prompt: list[int],
544 - tokenizer: TokenizerLike | None,
482 *,
483 start_idx: int = 0,
484 ) -> Generator[PromptTargetMatch]:
@@ -549,52 +486,15 @@ class ResolvedPromptUpdate:
486 target = self.target
487
488 if isinstance(target, PromptIndex):
552 - match_idx = target.get_match_index(tokenizer, prompt, start_idx)
489 + match_idx = target.get_match_index(prompt, start_idx)
490 if match_idx is not None:
491 yield PromptTargetMatch(match_idx, match_idx)
492
493 return
494
558 - target_token_ids = _seq2tokens(tokenizer, target)
559 -
560 - for match in iter_token_matches(prompt, target_token_ids, start_idx=start_idx):
495 + for match in iter_token_matches(prompt, target, start_idx=start_idx):
496 yield PromptTargetMatch(match.start_idx, match.end_idx)
497
563 - def iter_text_matches(
564 - self,
565 - prompt: str,
566 - tokenizer: TokenizerLike | None,
567 - *,
568 - start_idx: int = 0,
569 - ) -> Generator[PromptTargetMatch]:
570 - """Yield each instance of `self.target` found in `prompt`."""
571 - target = self.target
572 -
573 - if isinstance(target, PromptIndex):
574 - match_idx = target.get_match_index(tokenizer, prompt, start_idx)
575 - if match_idx is not None:
576 - yield PromptTargetMatch(match_idx, match_idx)
577 -
578 - return
579 -
580 - target_text = _seq2text(tokenizer, target)
581 -
582 - for match in re.finditer(re.escape(target_text), prompt, pos=start_idx):
583 - yield PromptTargetMatch(match.start(), match.end())
584 -
585 - def iter_matches(
586 - self,
587 - prompt: list[int] | str,
588 - tokenizer: TokenizerLike | None,
589 - *,
590 - start_idx: int = 0,
591 - ) -> Generator[PromptTargetMatch]:
592 - """Yield each instance of `self.target` found in `prompt`."""
593 - if isinstance(prompt, str):
594 - return self.iter_text_matches(prompt, tokenizer, start_idx=start_idx)
595 -
596 - return self.iter_token_matches(prompt, tokenizer, start_idx=start_idx)
597 -
498 def with_target(self, target: UpdateTarget):
499 return replace(self, target=target)
500
@@ -729,8 +629,6 @@ def _target_key(target: UpdateTarget) -> tuple[str, object]:
629 """Return a hashable key that preserves target matching semantics."""
630 if isinstance(target, PromptIndex):
631 return ("index", id(target))
732 - if isinstance(target, str):
733 - return ("text", target)
632
633 return ("tokens", tuple(target))
634
@@ -761,10 +659,16 @@ def _compile_prompt_update_queues(
659 return queues_by_signature
660
661
662 +_IterMatches: TypeAlias = Callable[
663 + [ResolvedPromptUpdate, int],
664 + Generator[PromptTargetMatch, None, None],
665 +]
666 +"""Yields each match of an update's target, given the search start index."""
667 +
668 +
669 def _find_queue_match(
670 queue: _UpdateQueue,
766 - prompt: _S,
767 - tokenizer: TokenizerLike | None,
671 + iter_matches: _IterMatches,
672 *,
673 start_idx: int,
674 mode: UpdateMode | None = None,
@@ -775,10 +679,7 @@ def _find_queue_match(
679 if mode is not None and update.mode != mode:
680 continue
681
778 - match = next(
779 - update.iter_matches(prompt, tokenizer, start_idx=start_idx),
780 - None,
781 - )
682 + match = next(iter_matches(update, start_idx), None)
683 if match is not None:
684 return match, update_idx
685
@@ -791,10 +692,9 @@ def _next_priority(queue: _UpdateQueue) -> int:
692 return priority
693
694
794 -def _plan_prompt_updates(
795 - prompt: _S,
695 +def _plan_prompt_updates_with(
696 mm_prompt_updates: "MultiModalPromptUpdates",
797 - tokenizer: TokenizerLike | None,
697 + iter_matches: _IterMatches,
698 ) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
699 """Plan non-overlapping prompt updates before rendering the output."""
700 queues = list(_compile_prompt_update_queues(mm_prompt_updates).values())
@@ -809,8 +709,7 @@ def _plan_prompt_updates(
709 for queue in queues:
710 queue_match = _find_queue_match(
711 queue,
812 - prompt,
813 - tokenizer,
712 + iter_matches,
713 start_idx=prev_end_idx,
714 )
715 if queue_match is not None:
@@ -835,8 +734,7 @@ def _plan_prompt_updates(
734
735 queue_match = _find_queue_match(
736 queue,
838 - prompt,
839 - tokenizer,
737 + iter_matches,
738 start_idx=prev_end_idx,
739 mode=mode,
740 )
@@ -887,6 +785,19 @@ def _plan_prompt_updates(
785 return planned_updates, result
786
787
788 +def _plan_prompt_updates(
789 + prompt: list[int],
790 + mm_prompt_updates: "MultiModalPromptUpdates",
791 +) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]:
792 + """Plan non-overlapping prompt updates before rendering the output."""
793 + return _plan_prompt_updates_with(
794 + mm_prompt_updates,
795 + lambda update, start_idx: update.iter_token_matches(
796 + prompt, start_idx=start_idx
797 + ),
798 + )
799 +
800 +
801 def _all_items_found(
802 mm_item_counts: dict[str, int],
803 mm_found_counts: dict[str, int],
@@ -898,22 +809,16 @@ def _all_items_found(
809
810
811 def _apply_matches(
901 - prompt: _S,
812 + prompt: list[int],
813 mm_prompt_updates: "MultiModalPromptUpdates",
903 - tokenizer: TokenizerLike | None,
904 -) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:
905 - out_seqs = list[str | list[int]]()
906 - matched_updates, result = _plan_prompt_updates(
907 - prompt,
908 - mm_prompt_updates,
909 - tokenizer,
910 - )
814 +) -> tuple[list[list[int]], "MultiModalPromptUpdatesApplyResult"]:
815 + out_seqs = list[list[int]]()
816 + matched_updates, result = _plan_prompt_updates(prompt, mm_prompt_updates)
817
818 prev_end_idx = 0
819 for matched_update in matched_updates:
820 update = matched_update.update
821 match = matched_update.match
916 - matched_content = update.content.full
822
823 if update.mode == UpdateMode.INSERT:
824 end_idx_to_insert = match.end_idx
@@ -923,22 +828,17 @@ def _apply_matches(
828 assert_never(update.mode)
829
830 out_seqs.append(prompt[prev_end_idx:end_idx_to_insert])
926 - out_seqs.append(
927 - _seq2text(tokenizer, matched_content)
928 - if isinstance(prompt, str)
929 - else _seq2tokens(tokenizer, matched_content)
930 - )
831 + out_seqs.append(update.content.full)
832 prev_end_idx = match.end_idx
833
834 out_seqs.append(prompt[prev_end_idx:])
835
935 - return cast(list[_S], out_seqs), result
836 + return out_seqs, result
837
838
839 def apply_token_matches(
840 prompt: list[int],
841 mm_prompt_updates: "MultiModalPromptUpdates",
941 - tokenizer: TokenizerLike | None,
842 ) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
843 """
844 Apply the updates in `mm_prompt_updates` to `prompt`.
@@ -947,7 +847,7 @@ def apply_token_matches(
847 the same placeholder tokens. In that case, the modality that
848 appears earlier in `mm_prompt_updates` takes priority.
849 """
950 - token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
850 + token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates)
851
852 return flatten_2d_lists(token_id_seqs), result
853
@@ -955,17 +855,12 @@ def apply_token_matches(
855 def _apply_token_matches_with_placeholders(
856 token_ids: list[int],
857 mm_prompt_updates: "MultiModalPromptUpdates",
958 - tokenizer: TokenizerLike | None,
858 ) -> tuple[
859 list[int],
860 "MultiModalPromptUpdatesApplyResult",
861 Mapping[str, list[PlaceholderFeaturesInfo]],
862 ]:
964 - matched_updates, result = _plan_prompt_updates(
965 - token_ids,
966 - mm_prompt_updates,
967 - tokenizer,
968 - )
863 + matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates)
864 placeholders: dict[str, list[PlaceholderFeaturesInfo]] = {
865 modality: [] for modality in mm_prompt_updates
866 }
@@ -975,7 +870,6 @@ def _apply_token_matches_with_placeholders(
870 for matched_update in matched_updates:
871 update = matched_update.update
872 match = matched_update.match
978 - matched_content = update.content.full
873
874 if update.mode == UpdateMode.INSERT:
875 end_idx_to_insert = match.end_idx
@@ -987,11 +881,11 @@ def _apply_token_matches_with_placeholders(
881 new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert])
882 start_idx = len(new_token_ids)
883
990 - tokens = _seq2tokens(tokenizer, matched_content)
884 + tokens = update.content.full
885 if tokens:
886 content_is_embed = update.content.is_embed
887 if content_is_embed is not None:
994 - content_is_embed = content_is_embed(tokenizer, matched_content)
888 + content_is_embed = content_is_embed(tokens)
889
890 placeholders[update.modality].append(
891 PlaceholderFeaturesInfo(
@@ -1010,50 +904,9 @@ def _apply_token_matches_with_placeholders(
904 return new_token_ids, result, placeholders
905
906
1013 -def apply_text_matches(
1014 - prompt: str,
1015 - mm_prompt_updates: "MultiModalPromptUpdates",
1016 - tokenizer: TokenizerLike | None,
1017 -) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:
1018 - """
1019 - Apply the updates in `mm_prompt_updates` to `prompt`.
1020 -
1021 - Matches are exclusive even when multiple modalities share
1022 - the same placeholder tokens. In that case, the modality that
1023 - appears earlier in `mm_prompt_updates` takes priority.
1024 - """
1025 - texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
1026 -
1027 - return "".join(texts), result
1028 -
1029 -
1030 -def apply_text_matches_as_segmented_tokens(
1031 - prompt: str,
1032 - mm_prompt_updates: "MultiModalPromptUpdates",
1033 - tokenizer: TokenizerLike | None,
1034 -) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:
1035 - """
1036 - Apply the updates in `mm_prompt_updates` to `prompt`.
1037 -
1038 - Matches are exclusive even when multiple modalities share
1039 - the same placeholder tokens. In that case, the modality that
1040 - appears earlier in `mm_prompt_updates` takes priority.
1041 -
1042 - Each segment is encoded separately instead of being joined into one
1043 - string and encoded in a single pass. Joining first would let BPE merge
1044 - tokens across a segment boundary, silently change how a text
1045 - (non-special-token) placeholder is tokenized.
1046 - """
1047 - texts, result = _apply_matches(prompt, mm_prompt_updates, tokenizer)
1048 - token_id_seqs = [_seq2tokens(tokenizer, text, use_cache=False) for text in texts]
1049 -
1050 - return flatten_2d_lists(token_id_seqs), result
1051 -
1052 -
907 def _iter_placeholders(
908 prompt: list[int],
909 mm_prompt_updates: "MultiModalPromptUpdates",
1056 - tokenizer: TokenizerLike | None,
910 ) -> Iterable[PlaceholderFeaturesInfo]:
911 """
912 Yield each set of placeholder tokens found in `prompt`.
@@ -1073,15 +926,15 @@ def _iter_placeholders(
926 prompt_len = len(prompt)
927 start_idx = 0
928
1076 - # The current (unfound) item's updates for each modality, with their
1077 - # content resolved to token ids; rebuilt whenever an item is found.
1078 - # Items are only resolved once the scan reaches them.
929 + # The current (unfound) item's updates for each modality; rebuilt
930 + # whenever an item is found. Items are only resolved once the scan
931 + # reaches them.
932 candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None
933
934 while start_idx < prompt_len:
935 if candidates is None:
936 candidates = [
1084 - (modality, update, _seq2tokens(tokenizer, update.content.full))
937 + (modality, update, update.content.full)
938 for modality, modality_updates in mm_prompt_updates.items()
939 if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0)
940 for update in modality_updates[item_idx_by_modality[modality]]
@@ -1107,7 +960,7 @@ def _iter_placeholders(
960 content = update.content
961 content_is_embed = content.is_embed
962 if content_is_embed is not None:
1110 - content_is_embed = content_is_embed(tokenizer, content.full)
963 + content_is_embed = content_is_embed(content.full)
964
965 yield PlaceholderFeaturesInfo(
966 modality=modality,
@@ -1134,9 +987,8 @@ def _iter_placeholders(
987 def find_mm_placeholders(
988 prompt: list[int],
989 mm_prompt_updates: "MultiModalPromptUpdates",
1137 - tokenizer: TokenizerLike | None,
990 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1139 - it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)
991 + it = _iter_placeholders(prompt, mm_prompt_updates)
992 return dict(full_groupby_modality(it))
993
994
@@ -1146,7 +998,7 @@ A collection of the `is_cached` flag for each item, with a similar structure as
998 [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
999 """
1000
1149 -MultiModalPromptUpdates = Mapping[str, list[Sequence[ResolvedPromptUpdate]]]
1001 +MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]]
1002 """
1003 A collection of prompt updates with a similar structure as
1004 [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems].
@@ -1283,9 +1135,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1135 new_token_ids: list[int],
1136 mm_prompt_updates: MultiModalPromptUpdates,
1137 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
1286 - tokenizer = self.info.get_tokenizer()
1287 -
1288 - return find_mm_placeholders(new_token_ids, mm_prompt_updates, tokenizer)
1138 + return find_mm_placeholders(new_token_ids, mm_prompt_updates)
1139
1140 def _get_hf_mm_data(
1141 self,
@@ -1606,8 +1456,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1456 prompt: list[int],
1457 mm_prompt_updates: MultiModalPromptUpdates,
1458 ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
1609 - tokenizer = self.info.get_tokenizer()
1610 - return apply_token_matches(prompt, mm_prompt_updates, tokenizer)
1459 + return apply_token_matches(prompt, mm_prompt_updates)
1460
1461 def _apply_token_matches_with_placeholders(
1462 self,
@@ -1618,31 +1467,105 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1467 MultiModalPromptUpdatesApplyResult,
1468 Mapping[str, list[PlaceholderFeaturesInfo]],
1469 ]:
1621 - tokenizer = self.info.get_tokenizer()
1470 return _apply_token_matches_with_placeholders(
1471 token_ids,
1472 mm_prompt_updates,
1625 - tokenizer,
1473 )
1474
1628 - def _apply_text_matches(
1475 + def _iter_text_matches(
1476 self,
1630 - prompt: str,
1631 - mm_prompt_updates: MultiModalPromptUpdates,
1632 - ) -> tuple[str, MultiModalPromptUpdatesApplyResult]:
1633 - tokenizer = self.info.get_tokenizer()
1634 - return apply_text_matches(prompt, mm_prompt_updates, tokenizer)
1477 + update: ResolvedPromptUpdate,
1478 + prompt_text: str,
1479 + tokenizer: TokenizerLike,
1480 + *,
1481 + start_idx: int = 0,
1482 + ) -> Generator[PromptTargetMatch, None, None]:
1483 + """Yield each instance of `update.target` found in `prompt_text`."""
1484 + target = update.target
1485 +
1486 + if isinstance(target, PromptIndex):
1487 + match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx)
1488 + if match_idx is not None:
1489 + yield PromptTargetMatch(match_idx, match_idx)
1490
1636 - def _apply_text_matches_as_segmented_tokens(
1491 + return
1492 +
1493 + target_text = tokenizer.decode(target)
1494 + if not target_text:
1495 + return
1496 +
1497 + idx = prompt_text.find(target_text, start_idx)
1498 + while idx != -1:
1499 + end_idx = idx + len(target_text)
1500 + yield PromptTargetMatch(idx, end_idx)
1501 + idx = prompt_text.find(target_text, end_idx)
1502 +
1503 + def _apply_prompt_updates_via_text(
1504 self,
1638 - prompt: str,
1505 + token_ids: list[int],
1506 mm_prompt_updates: MultiModalPromptUpdates,
1507 + *,
1508 + encode_segments_separately: bool = False,
1509 ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]:
1510 + """
1511 + Apply multi-modal prompt updates in text space, then encode the
1512 + result.
1513 +
1514 + A non-special-token target may be tokenized differently inside the
1515 + prompt because the tokens may go across the boundaries of the target
1516 + (e.g. when searching for "foo" in "food", if "food" itself makes up
1517 + a token, then the token ID of "foo" will not appear at all). Since
1518 + it is inefficient to search for all possible tokenizations of the
1519 + target in the prompt, we instead perform text-based updates on the
1520 + decoded token IDs, then encode them back.
1521 + """
1522 tokenizer = self.info.get_tokenizer()
1642 - return apply_text_matches_as_segmented_tokens(
1643 - prompt, mm_prompt_updates, tokenizer
1523 + prompt_text = tokenizer.decode(token_ids)
1524 +
1525 + matched_updates, result = _plan_prompt_updates_with(
1526 + mm_prompt_updates,
1527 + lambda update, start_idx: self._iter_text_matches(
1528 + update, prompt_text, tokenizer, start_idx=start_idx
1529 + ),
1530 )
1531
1532 + out_texts = list[str]()
1533 + prev_end_idx = 0
1534 + for matched_update in matched_updates:
1535 + update = matched_update.update
1536 + match = matched_update.match
1537 +
1538 + if update.mode == UpdateMode.INSERT:
1539 + end_idx_to_insert = match.end_idx
1540 + elif update.mode == UpdateMode.REPLACE:
1541 + end_idx_to_insert = match.start_idx
1542 + else:
1543 + assert_never(update.mode)
1544 +
1545 + out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert])
1546 + out_texts.append(tokenizer.decode(update.content.full))
1547 + prev_end_idx = match.end_idx
1548 +
1549 + out_texts.append(prompt_text[prev_end_idx:])
1550 +
1551 + if encode_segments_separately:
1552 + # Encode each segment separately instead of joining into one
1553 + # text and encoding it in a single pass. Joining first would
1554 + # let BPE merge tokens across a segment boundary, silently
1555 + # changing how a non-special-token placeholder is tokenized.
1556 + new_token_ids = flatten_2d_lists(
1557 + [
1558 + cached_encode(tokenizer, text, add_special_tokens=False)
1559 + for text in out_texts
1560 + ]
1561 + )
1562 + else:
1563 + new_token_ids = cached_encode(
1564 + tokenizer, "".join(out_texts), add_special_tokens=False
1565 + )
1566 +
1567 + return new_token_ids, result
1568 +
1569 def _matched_updates_from_result(
1570 self,
1571 mm_prompt_updates: MultiModalPromptUpdates,
@@ -1668,8 +1591,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1591 mm_prompt_updates: MultiModalPromptUpdates,
1592 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
1593 """Apply multi-modal prompt updates to token IDs."""
1671 - tokenizer = self.info.get_tokenizer()
1672 -
1594 new_token_ids, match_result, placeholders = (
1595 self._apply_token_matches_with_placeholders(
1596 token_ids,
@@ -1688,23 +1609,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1609 }
1610 return new_token_ids, placeholders
1611
1691 - # If the search text does not represent a special token,
1692 - # it may have different token IDs in the prompt, because
1693 - # the tokens may go across the boundaries of the search text.
1694 - # ----
1695 - # e.g. when searching for "foo" in "food", if "food" itself makes
1696 - # up a token, then the token ID of "foo" will not appear at all
1697 - # ----
1698 - # Since it is inefficient to search for all possible tokenizations
1699 - # of the search text in the prompt, we instead perform string-based
1700 - # updates on the decoded token IDs, then encode them back.
1701 - new_text, match_result = self._apply_text_matches(
1702 - _seq2text(tokenizer, token_ids, use_cache=False),
1612 + # A non-special-token target may be tokenized differently inside
1613 + # the prompt, so fall back to performing the updates on the
1614 + # decoded text, then encoding the result back.
1615 + new_token_ids, match_result = self._apply_prompt_updates_via_text(
1616 + token_ids,
1617 mm_prompt_updates,
1618 )
1619
1706 - new_token_ids = _seq2tokens(tokenizer, new_text, use_cache=False)
1707 -
1620 placeholders = self._find_mm_placeholders(
1621 new_token_ids,
1622 self._matched_updates_from_result(mm_prompt_updates, match_result),
vllm/renderers/hf.py
+2 -6
@@ -157,7 +157,7 @@ def _expand_prompt_embeds_placeholders(
157 `token_ids` is replaced with a consecutive span of
158 `tensor.shape[0]` copies, following tensors in order.
159 """
160 - expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
160 + expanded, _ = apply_token_matches(token_ids, mm_prompt_updates)
161 return expanded
162
163
@@ -171,11 +171,7 @@ def _build_prompt_embeds_positions(
171 Expects `token_ids` to already contain expanded N-token spans.
172 Returns `[(start_idx, length), ...]` aligned with the tensors.
173 """
174 - placeholders = find_mm_placeholders(
175 - prompt=token_ids,
176 - mm_prompt_updates=mm_prompt_updates,
177 - tokenizer=None,
178 - )
174 + placeholders = find_mm_placeholders(token_ids, mm_prompt_updates)
175 features = placeholders.get("prompt_embeds", [])
176
177 if len(features) != num_tensors:
vllm/transformers_utils/processors/internvl.py
+24 -7
@@ -21,7 +21,7 @@ from transformers import (
21 from transformers.processing_utils import ProcessorMixin
22
23 from vllm.multimodal.image import convert_image_mode
24 -from vllm.multimodal.processing import PromptUpdateDetails
24 +from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
25 from vllm.tokenizers.hf import HfTokenizer
26
27 IMAGENET_MEAN = (0.485, 0.456, 0.406)
@@ -444,7 +444,7 @@ class InternVLProcessor(ProcessorMixin):
444 self,
445 num_patches: int | None,
446 num_features: int | None = None,
447 - ) -> PromptUpdateDetails[str]:
447 + ) -> PromptUpdateDetails:
448 if num_patches is None:
449 assert num_features is not None
450 else:
@@ -453,9 +453,11 @@ class InternVLProcessor(ProcessorMixin):
453 repl_features = self.ctx_image_token * num_features
454 repl_full = self.start_image_token + repl_features + self.end_image_token
455
456 - return PromptUpdateDetails.select_text(repl_full, self.ctx_image_token)
456 + full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
457
458 - def get_video_repl(self, num_patches: int) -> PromptUpdateDetails[str]:
458 + return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
459 +
460 + def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
461 assert self.ctx_video_token is not None
462
463 repl_features = self.ctx_video_token * self.image_seq_length
@@ -467,7 +469,10 @@ class InternVLProcessor(ProcessorMixin):
469 [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
470 )
471
470 - return PromptUpdateDetails.select_text(repl_full, self.ctx_video_token)
472 + full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
473 + assert self.ctx_video_token_id is not None
474 +
475 + return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
476
477 def __call__(
478 self,
@@ -523,7 +528,13 @@ class InternVLProcessor(ProcessorMixin):
528 while image_token in new_prompt:
529 new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
530 image_repl = self.get_image_repl(image_num_patches[image_index])
526 - replace_strings.append(image_repl.full)
531 + # Convert the token IDs back to text for the
532 + # string-based replacement below
533 + replace_strings.append(
534 + self.tokenizer.decode(
535 + image_repl.full, skip_special_tokens=False
536 + )
537 + )
538 image_index += 1
539
540 while "<placeholder>" in new_prompt:
@@ -548,7 +559,13 @@ class InternVLProcessor(ProcessorMixin):
559 while video_token in new_prompt:
560 new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
561 video_repl = self.get_video_repl(video_num_patches[video_index])
551 - replace_strings.append(video_repl.full)
562 + # Convert the token IDs back to text for the
563 + # string-based replacement below
564 + replace_strings.append(
565 + self.tokenizer.decode(
566 + video_repl.full, skip_special_tokens=False
567 + )
568 + )
569 video_index += 1
570
571 while "<placeholder>" in new_prompt:
vllm/transformers_utils/processors/nano_nemotron_vl.py
+28 -9
@@ -24,7 +24,7 @@ from transformers import BatchFeature, PretrainedConfig, TensorType
24
25 from vllm.model_executor.models.parakeet import ParakeetExtractor
26 from vllm.multimodal.inputs import AudioItem
27 -from vllm.multimodal.processing.processor import PromptUpdateDetails
27 +from vllm.multimodal.processing.processor import PromptUpdateDetails, cached_encode
28 from vllm.multimodal.video_prune.evs import compute_retained_tokens_count
29 from vllm.platforms import current_platform
30 from vllm.tokenizers.hf import HfTokenizer
@@ -633,7 +633,7 @@ class BaseNanoNemotronVLProcessor(ABC):
633 self,
634 feature_size: int,
635 num_patches: int | None,
636 - ) -> PromptUpdateDetails[str]:
636 + ) -> PromptUpdateDetails:
637 raise NotImplementedError
638
639 def get_num_image_tokens(
@@ -730,7 +730,12 @@ class BaseNanoNemotronVLProcessor(ABC):
730 zip(num_tokens_per_image, image_num_patches, strict=True)
731 ):
732 image_repl = self.get_image_repl(feature_size, num_patches)
733 - parts[i] = parts[i].replace("<image>", image_repl.full)
733 + # image_repl.full is a list of token IDs
734 + # Convert token IDs back to text for the HF processor flow
735 + image_repl_text = self.tokenizer.decode(
736 + image_repl.full, skip_special_tokens=False
737 + )
738 + parts[i] = parts[i].replace("<image>", image_repl_text)
739 text = ["".join(parts)]
740
741 return text, image_inputs
@@ -1001,7 +1006,11 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
1006 for idx, part in enumerate(parts):
1007 if part == AUDIO_CONTEXT:
1008 audio_repl = self.get_audio_repl(audios[audio_index])
1004 - parts[idx] = audio_repl.full
1009 + # audio_repl.full is a list of token IDs
1010 + # Convert token IDs back to text for the HF processor flow
1011 + parts[idx] = self.tokenizer.decode(
1012 + audio_repl.full, skip_special_tokens=False
1013 + )
1014 audio_index += 1
1015 text = ["".join(parts)]
1016 audio_inputs = extractor(audios)
@@ -1078,20 +1087,30 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
1087 self,
1088 feature_size: int,
1089 num_patches: int | None,
1081 - ) -> PromptUpdateDetails[str]:
1090 + ) -> PromptUpdateDetails:
1091 repl_features = IMG_CONTEXT * feature_size
1092 repl_full = IMG_START + repl_features + IMG_END
1093
1085 - return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)
1094 + full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
1095 +
1096 + return PromptUpdateDetails.select_token_ids(
1097 + full_ids, self._img_context_token_ids
1098 + )
1099
1100 def get_audio_repl(
1101 self,
1102 audio: npt.NDArray,
1090 - ) -> PromptUpdateDetails[str]:
1103 + ) -> PromptUpdateDetails:
1104 assert self.audio_extractor is not None
1105 num_tokens = self.audio_extractor.audio_token_count(len(audio))
1106 repl_full = f"{AUDIO_START}{AUDIO_CONTEXT * num_tokens}{AUDIO_END}"
1094 - return PromptUpdateDetails.select_text(repl_full, AUDIO_CONTEXT)
1107 +
1108 + full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
1109 + ctx_token_ids = cached_encode(
1110 + self.tokenizer, AUDIO_CONTEXT, add_special_tokens=False
1111 + )
1112 +
1113 + return PromptUpdateDetails.select_token_ids(full_ids, ctx_token_ids)
1114
1115 @classmethod
1116 def get_video_repl(
@@ -1105,7 +1124,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
1124 img_end_token_ids: list[int],
1125 img_context_token_ids: list[int],
1126 video_temporal_patch_size: int = 1,
1108 - ) -> PromptUpdateDetails[list[int]]:
1127 + ) -> PromptUpdateDetails:
1128 """
1129 Build prompt replacement for a video.
1130 The replacement returned is not actually used to replace the placeholder
vllm/transformers_utils/processors/nvlm_d.py
+5 -3
@@ -7,7 +7,7 @@
7 # Copyright (c) 2024 NVIDIA
8 # Licensed under Apache 2.0 License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from vllm.multimodal.processing import PromptUpdateDetails
10 +from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
11 from vllm.tokenizers.hf import HfTokenizer
12
13 from .internvl import InternVLImageProcessor, InternVLProcessor
@@ -37,7 +37,7 @@ class NVLMProcessor(InternVLProcessor):
37 self,
38 num_patches: int | None,
39 num_features: int | None = None,
40 - ) -> PromptUpdateDetails[str]:
40 + ) -> PromptUpdateDetails:
41 if num_patches is None:
42 raise NotImplementedError("Embedding inputs are not supported")
43
@@ -58,4 +58,6 @@ class NVLMProcessor(InternVLProcessor):
58 # when trying to find "<tile" as a subsequence of "<Image><tile"
59 repl = self.start_image_token + features + self.end_image_token
60
61 - return PromptUpdateDetails.select_text(repl, self.ctx_image_token)
61 + full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False)
62 +
63 + return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)