@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f7bee5c815
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[VLM][Bugfix] Enable specifying prompt target via index (#14038)

Cyrus Leung committed Feb 28, 2025 at 23:35 UTC f7bee5c8158fb2a6b4beb90f881298c143ea0c51
5 files changed +432 -59
tests/multimodal/test_processing.py
+256 -2
@@ -14,8 +14,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
14 # yapf conflicts with isort for this block
15 # yapf: disable
16 from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
17 - PromptInsertion, PromptReplacement,
18 - apply_text_matches,
17 + PromptIndexTargets, PromptInsertion,
18 + PromptReplacement, apply_text_matches,
19 apply_token_matches,
20 find_mm_placeholders,
21 find_text_matches, find_token_matches,
@@ -98,10 +98,20 @@ def test_iter_token_matches(token_ids, match_ids, expected):
98 {
99 "pattern_1": [],
100 "pattern_2": [32000],
101 + "pattern_3": PromptIndexTargets.start(),
102 + "pattern_4": PromptIndexTargets.prefix([32000]),
103 + "pattern_5": PromptIndexTargets.end(),
104 },
105 {
106 "pattern_1": [],
107 "pattern_2": [],
108 + "pattern_3": [
109 + { "start_idx": 0, "end_idx": 0 },
110 + ],
111 + "pattern_4": [],
112 + "pattern_5": [
113 + { "start_idx": 0, "end_idx": 0 },
114 + ],
115 },
116 ),
117 (
@@ -110,6 +120,9 @@ def test_iter_token_matches(token_ids, match_ids, expected):
120 "pattern_1": [32000],
121 "pattern_2": [32000, 32000],
122 "pattern_3": [32000, 32000, 32000],
123 + "pattern_4": PromptIndexTargets.start(),
124 + "pattern_5": PromptIndexTargets.prefix([32000]),
125 + "pattern_6": PromptIndexTargets.end(),
126 },
127 {
128 "pattern_1": [
@@ -125,6 +138,15 @@ def test_iter_token_matches(token_ids, match_ids, expected):
138 "pattern_3": [
139 { "start_idx": 0, "end_idx": 3 },
140 ],
141 + "pattern_4": [
142 + { "start_idx": 0, "end_idx": 0 },
143 + ],
144 + "pattern_5": [
145 + { "start_idx": 1, "end_idx": 1 },
146 + ],
147 + "pattern_6": [
148 + { "start_idx": 4, "end_idx": 4 },
149 + ],
150 },
151 ),
152 (
@@ -133,6 +155,9 @@ def test_iter_token_matches(token_ids, match_ids, expected):
155 "pattern_1": [28747, 32000],
156 "pattern_2": [28747, 32000, 32000, 32000],
157 "pattern_3": [28747, 0, 32000],
158 + "pattern_4": PromptIndexTargets.start(),
159 + "pattern_5": PromptIndexTargets.prefix([28747, 32000]),
160 + "pattern_6": PromptIndexTargets.end(),
161 },
162 {
163 "pattern_1": [
@@ -143,6 +168,13 @@ def test_iter_token_matches(token_ids, match_ids, expected):
168 { "start_idx": 1, "end_idx": 5 },
169 ],
170 "pattern_3": [],
171 + "pattern_4": [
172 + { "start_idx": 0, "end_idx": 0 },
173 + ],
174 + "pattern_5": [],
175 + "pattern_6": [
176 + { "start_idx": 10, "end_idx": 10 },
177 + ],
178 },
179 ),
180 ],
@@ -189,10 +221,20 @@ def test_find_token_matches(
221 {
222 "pattern_1": "",
223 "pattern_2": "<image>",
224 + "pattern_3": PromptIndexTargets.start(),
225 + "pattern_4": PromptIndexTargets.prefix("<image>"),
226 + "pattern_5": PromptIndexTargets.end(),
227 },
228 {
229 "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
230 "pattern_2": [],
231 + "pattern_3": [
232 + { "start_idx": 0, "end_idx": 0 },
233 + ],
234 + "pattern_4": [],
235 + "pattern_5": [
236 + { "start_idx": 0, "end_idx": 0 },
237 + ],
238 }
239 ),
240 (
@@ -201,6 +243,9 @@ def test_find_token_matches(
243 "pattern_1": "<image>",
244 "pattern_2": "<image><image>",
245 "pattern_3": "<image><image><image>",
246 + "pattern_4": PromptIndexTargets.start(),
247 + "pattern_5": PromptIndexTargets.prefix("<image>"),
248 + "pattern_6": PromptIndexTargets.end(),
249 },
250 {
251 "pattern_1": [
@@ -216,6 +261,15 @@ def test_find_token_matches(
261 "pattern_3": [
262 { "start_idx": 0, "end_idx": 21 },
263 ],
264 + "pattern_4": [
265 + { "start_idx": 0, "end_idx": 0 },
266 + ],
267 + "pattern_5": [
268 + { "start_idx": 7, "end_idx": 7 },
269 + ],
270 + "pattern_6": [
271 + { "start_idx": 28, "end_idx": 28 },
272 + ],
273 },
274 ),
275 (
@@ -224,6 +278,9 @@ def test_find_token_matches(
278 "pattern_1": "Image:<image>",
279 "pattern_2": "Image:<image><image><image>",
280 "pattern_3": "Image:<unk><image>",
281 + "pattern_4": PromptIndexTargets.start(),
282 + "pattern_5": PromptIndexTargets.prefix("Image:<image>"),
283 + "pattern_6": PromptIndexTargets.end(),
284 },
285 {
286 "pattern_1": [
@@ -234,6 +291,15 @@ def test_find_token_matches(
291 { "start_idx": 0, "end_idx": 27 },
292 ],
293 "pattern_3": [],
294 + "pattern_4": [
295 + { "start_idx": 0, "end_idx": 0 },
296 + ],
297 + "pattern_5": [
298 + { "start_idx": 13, "end_idx": 13 },
299 + ],
300 + "pattern_6": [
301 + { "start_idx": 48, "end_idx": 48 },
302 + ],
303 },
304 ),
305 # Test regex escape
@@ -325,6 +391,100 @@ def test_find_text_matches(
391 },
392 },
393 ),
394 + # Test index targets
395 + (
396 + "",
397 + {
398 + "pattern_1": PromptIndexTargets.start(),
399 + "pattern_2": PromptIndexTargets.prefix("<image>"),
400 + "pattern_3": PromptIndexTargets.end(),
401 + },
402 + {
403 + "pattern_1": "1",
404 + "pattern_2": "2",
405 + "pattern_3": "3",
406 + },
407 + {
408 + PromptInsertion: {
409 + 0: "",
410 + 1: "13",
411 + 2: "1133",
412 + },
413 + PromptReplacement: {
414 + 0: "",
415 + 1: "13",
416 + 2: "1133",
417 + },
418 + },
419 + ),
420 + (
421 + "<image>",
422 + {
423 + "pattern_1": PromptIndexTargets.start(),
424 + "pattern_2": PromptIndexTargets.prefix("<image>"),
425 + "pattern_3": PromptIndexTargets.end(),
426 + },
427 + {
428 + "pattern_1": "1",
429 + "pattern_2": "2",
430 + "pattern_3": "3",
431 + },
432 + {
433 + PromptInsertion: {
434 + 0: "<image>",
435 + 1: "1<image>23",
436 + 2: "11<image>2233",
437 + },
438 + PromptReplacement: {
439 + 0: "<image>",
440 + 1: "1<image>23",
441 + 2: "11<image>2233",
442 + },
443 + },
444 + ),
445 + # Test different replacement per item
446 + (
447 + "<image><image><image>",
448 + {
449 + "pattern_1": "<image>",
450 + },
451 + {
452 + "pattern_1": lambda idx: str(idx + 1),
453 + },
454 + {
455 + PromptInsertion: {
456 + 0: "<image><image><image>",
457 + 1: "<image>1<image><image>",
458 + 2: "<image>12<image><image>",
459 + },
460 + PromptReplacement: {
461 + 0: "<image><image><image>",
462 + 1: "1<image><image>",
463 + 2: "12<image>",
464 + },
465 + },
466 + ),
467 + (
468 + "<image><image><image>",
469 + {
470 + "pattern_1": PromptIndexTargets.prefix("<image>"),
471 + },
472 + {
473 + "pattern_1": lambda idx: str(idx + 1),
474 + },
475 + {
476 + PromptInsertion: {
477 + 0: "<image><image><image>",
478 + 1: "<image>1<image><image>",
479 + 2: "<image>12<image><image>",
480 + },
481 + PromptReplacement: {
482 + 0: "<image><image><image>",
483 + 1: "<image>1<image><image>",
484 + 2: "<image>12<image><image>",
485 + },
486 + },
487 + ),
488 ]
489 )
490 # yapf: enable
@@ -405,6 +565,100 @@ def test_find_update_text(
565 },
566 },
567 ),
568 + # Test index targets
569 + (
570 + [],
571 + {
572 + "pattern_1": PromptIndexTargets.start(),
573 + "pattern_2": PromptIndexTargets.prefix([32000]),
574 + "pattern_3": PromptIndexTargets.end(),
575 + },
576 + {
577 + "pattern_1": [-1],
578 + "pattern_2": [-2],
579 + "pattern_3": [-3],
580 + },
581 + {
582 + PromptInsertion: {
583 + 0: [],
584 + 1: [-1, -3],
585 + 2: [-1, -1, -3, -3],
586 + },
587 + PromptReplacement: {
588 + 0: [],
589 + 1: [-1, -3],
590 + 2: [-1, -1, -3, -3],
591 + },
592 + },
593 + ),
594 + (
595 + [32000],
596 + {
597 + "pattern_1": PromptIndexTargets.start(),
598 + "pattern_2": PromptIndexTargets.prefix([32000]),
599 + "pattern_3": PromptIndexTargets.end(),
600 + },
601 + {
602 + "pattern_1": [-1],
603 + "pattern_2": [-2],
604 + "pattern_3": [-3],
605 + },
606 + {
607 + PromptInsertion: {
608 + 0: [32000],
609 + 1: [-1, 32000, -2, -3],
610 + 2: [-1, -1, 32000, -2, -2, -3, -3],
611 + },
612 + PromptReplacement: {
613 + 0: [32000],
614 + 1: [-1, 32000, -2, -3],
615 + 2: [-1, -1, 32000, -2, -2, -3, -3],
616 + },
617 + },
618 + ),
619 + # Test different replacement per item
620 + (
621 + [32000, 32000, 32000],
622 + {
623 + "pattern_1": [32000],
624 + },
625 + {
626 + "pattern_1": lambda idx: [-(idx + 1)],
627 + },
628 + {
629 + PromptInsertion: {
630 + 0: [32000, 32000, 32000],
631 + 1: [32000, -1, 32000, 32000],
632 + 2: [32000, -1, -2, 32000, 32000],
633 + },
634 + PromptReplacement: {
635 + 0: [32000, 32000, 32000],
636 + 1: [-1, 32000, 32000],
637 + 2: [-1, -2, 32000],
638 + },
639 + },
640 + ),
641 + (
642 + [32000, 32000, 32000],
643 + {
644 + "pattern_1": PromptIndexTargets.prefix([32000]),
645 + },
646 + {
647 + "pattern_1": lambda idx: [-(idx + 1)],
648 + },
649 + {
650 + PromptInsertion: {
651 + 0: [32000, 32000, 32000],
652 + 1: [32000, -1, 32000, 32000],
653 + 2: [32000, -1, -2, 32000, 32000],
654 + },
655 + PromptReplacement: {
656 + 0: [32000, 32000, 32000],
657 + 1: [32000, -1, 32000, 32000],
658 + 2: [32000, -1, -2, 32000, 32000],
659 + },
660 + },
661 + ),
662 ]
663 )
664 # yapf: enable
vllm/model_executor/models/blip2.py
+3 -3
@@ -19,8 +19,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19 NestedTensors)
20 from vllm.multimodal.parse import MultiModalDataItems
21 from vllm.multimodal.processing import (BaseMultiModalProcessor,
22 - BaseProcessingInfo, PromptInsertion,
23 - PromptUpdate)
22 + BaseProcessingInfo, PromptIndexTargets,
23 + PromptInsertion, PromptUpdate)
24 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25 from vllm.sequence import IntermediateTensors
26
@@ -490,7 +490,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
490 return [
491 PromptInsertion(
492 modality="image",
493 - target="",
493 + target=PromptIndexTargets.start(),
494 insertion=image_tokens,
495 )
496 ]
vllm/model_executor/models/florence2.py
+3 -2
@@ -25,7 +25,8 @@ from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25 from vllm.multimodal.parse import MultiModalDataDict, MultiModalDataItems
26 from vllm.multimodal.processing import (BaseProcessingInfo,
27 EncDecMultiModalProcessor,
28 - PromptInsertion, PromptUpdate)
28 + PromptIndexTargets, PromptInsertion,
29 + PromptUpdate)
30 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 from vllm.sequence import IntermediateTensors
32
@@ -864,7 +865,7 @@ class Florence2MultiModalProcessor(
865 return [
866 PromptInsertion(
867 modality="image",
867 - target="",
868 + target=PromptIndexTargets.start(),
869 insertion=image_tokens,
870 )
871 ]
vllm/model_executor/models/molmo.py
+3 -3
@@ -46,8 +46,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
46 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47 MultiModalDataItems)
48 from vllm.multimodal.processing import (BaseMultiModalProcessor,
49 - BaseProcessingInfo, PromptInsertion,
50 - PromptUpdate)
49 + BaseProcessingInfo, PromptIndexTargets,
50 + PromptInsertion, PromptUpdate)
51 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52 from vllm.sequence import IntermediateTensors
53 from vllm.utils import JSONTree, json_map_leaves
@@ -1371,7 +1371,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1371 return [
1372 PromptInsertion(
1373 modality="image",
1374 - target="<|endoftext|>",
1374 + target=PromptIndexTargets.prefix("<|endoftext|>"),
1375 insertion=get_insertion_molmo,
1376 )
1377 ]
vllm/multimodal/processing.py
+167 -49
@@ -8,7 +8,6 @@ from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
8 from dataclasses import dataclass, field
9 from enum import Enum
10 from functools import lru_cache
11 -from itertools import groupby
11 from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
12 TypeVar, Union, cast)
13
@@ -40,6 +39,65 @@ PromptSeq = Union[str, list[int]]
39 """A token sequence (list of token IDs) or text."""
40
41
42 +@dataclass
43 +class PromptIndex:
44 + """Resolves to an index in the prompt."""
45 + get_match_index: Callable[[AnyTokenizer, PromptSeq], Optional[int]]
46 +
47 +
48 +class PromptIndexTargets:
49 +
50 + @staticmethod
51 + def start() -> PromptIndex:
52 + """
53 + Resolves to the start of the prompt (before the first token).
54 +
55 + This results in a match even if the prompt is empty.
56 + """
57 + return PromptIndex(lambda tok, prompt: 0)
58 +
59 + @staticmethod
60 + def prefix(seq: PromptSeq) -> PromptIndex:
61 + """
62 + Resolves to a location in the prompt after the given prefix.
63 + """
64 +
65 + def get_match_index(
66 + tokenizer: AnyTokenizer,
67 + prompt: PromptSeq,
68 + ) -> Optional[int]:
69 + prefix = seq
70 +
71 + if isinstance(prompt, str):
72 + if not isinstance(prefix, str):
73 + # Make both `str`
74 + prefix = decode_tokens(tokenizer, prefix)
75 + else:
76 + if isinstance(prefix, str):
77 + # Make both `list[int]`
78 + prefix = encode_tokens(tokenizer, prefix)
79 +
80 + match_idx = len(prefix)
81 + return match_idx if prompt[:match_idx] == prefix else None
82 +
83 + return PromptIndex(get_match_index)
84 +
85 + @staticmethod
86 + def end() -> PromptIndex:
87 + """
88 + Resolves to the end of the prompt (after the last token).
89 +
90 + This results in a match even if the prompt is empty.
91 + """
92 + return PromptIndex(lambda tok, prompt: len(prompt))
93 +
94 +
95 +PromptTarget = Union[PromptSeq, PromptIndex]
96 +"""
97 +The token sequence or text to update.
98 +"""
99 +
100 +
101 @dataclass
102 class PromptUpdateDetails:
103 """Details about the token sequence or text that are part of the update."""
@@ -84,7 +142,7 @@ class UpdateMode(str, Enum):
142
143
144 @dataclass
87 -class PromptUpdate:
145 +class PromptUpdate(ABC):
146 """
147 Defines how to update a prompt with placeholder tokens.
148 """
@@ -92,7 +150,7 @@ class PromptUpdate:
150 modality: str
151 """The modality for which the update is made."""
152
95 - target: PromptSeq
153 + target: PromptTarget
154 """The token sequence (or text) to update."""
155
156 @property
@@ -122,24 +180,43 @@ class PromptInsertion(PromptUpdate):
180 Example:
181
182 For each image, insert a number of ``<image>`` feature placeholders
125 - equal to the feature size of the vision encoder at the start of the
126 - prompt:
183 + equal to the feature size of the vision encoder after the ``<s>`` token:
184
185 .. code-block:: python
186
187 PromptInsertion(
188 modality="image",
132 - target="",
189 + target="<s>",
190 insertion="<image>" * image_feature_size,
191 )
192
136 - As above, but insert after the ``<s>`` token:
193 + Insert these tokens at the start of the prompt:
194
195 .. code-block:: python
196
197 PromptInsertion(
198 modality="image",
142 - target="<s>",
199 + target=PromptIndexTargets.start(),
200 + insertion="<image>" * image_feature_size,
201 + )
202 +
203 + Insert these tokens after a prefix ``Images:``:
204 +
205 + .. code-block:: python
206 +
207 + PromptInsertion(
208 + modality="image",
209 + target=PromptIndexTargets.prefix("Images:"),
210 + insertion="<image>" * image_feature_size,
211 + )
212 +
213 + Insert these tokens at the end of the prompt:
214 +
215 + .. code-block:: python
216 +
217 + PromptInsertion(
218 + modality="image",
219 + target=PromptIndexTargets.end(),
220 insertion="<image>" * image_feature_size,
221 )
222 """
@@ -345,10 +422,14 @@ class BoundPromptUpdate:
422 return self._origin.modality
423
424 @property
348 - def target(self) -> _BoundPromptSequence:
425 + def target(self) -> Union[_BoundPromptSequence, PromptIndex]:
426 """The token sequence (or text) to update."""
350 - return _BoundPromptSequence.from_seq(self.tokenizer,
351 - self._origin.target)
427 + target = self._origin.target
428 +
429 + if isinstance(target, PromptIndex):
430 + return target
431 +
432 + return _BoundPromptSequence.from_seq(self.tokenizer, target)
433
434 @property
435 def content(self) -> PromptUpdateContent:
@@ -447,6 +528,19 @@ class _PromptTargetMatch(ABC):
528 f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
529
530
531 +@dataclass(repr=False)
532 +class _PromptTargetIndexMatch(_PromptTargetMatch):
533 + match_idx: int
534 +
535 + @property
536 + def start_idx(self) -> int:
537 + return self.match_idx
538 +
539 + @property
540 + def end_idx(self) -> int:
541 + return self.match_idx
542 +
543 +
544 @dataclass(repr=False)
545 class _PromptTargetTokenMatch(_PromptTargetMatch):
546 match: _TokenMatch
@@ -496,9 +590,24 @@ def find_token_matches(
590 prompt_updates: Sequence[BoundPromptUpdate],
591 ) -> Sequence[_PromptTargetMatch]:
592 """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
593 +
594 + def get_matches(update: BoundPromptUpdate):
595 + target = update.target
596 +
597 + if isinstance(target, PromptIndex):
598 + match_idx = target.get_match_index(update.tokenizer, prompt)
599 + if match_idx is None:
600 + return []
601 +
602 + return [_PromptTargetIndexMatch(update, match_idx)]
603 +
604 + return [
605 + _PromptTargetTokenMatch(update, match)
606 + for match in iter_token_matches(prompt, target.token_ids)
607 + ]
608 +
609 return [
500 - _PromptTargetTokenMatch(update, match) for update in prompt_updates
501 - for match in iter_token_matches(prompt, update.target.token_ids)
610 + match for update in prompt_updates for match in get_matches(update)
611 ]
612
613
@@ -507,9 +616,24 @@ def find_text_matches(
616 prompt_updates: Sequence[BoundPromptUpdate],
617 ) -> Sequence[_PromptTargetMatch]:
618 """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
619 +
620 + def get_matches(update: BoundPromptUpdate):
621 + target = update.target
622 +
623 + if isinstance(target, PromptIndex):
624 + match_idx = target.get_match_index(update.tokenizer, prompt)
625 + if match_idx is None:
626 + return []
627 +
628 + return [_PromptTargetIndexMatch(update, match_idx)]
629 +
630 + return [
631 + _PromptTargetTextMatch(update, match)
632 + for match in re.finditer(re.escape(target.text), prompt)
633 + ]
634 +
635 return [
511 - _PromptTargetTextMatch(update, match) for update in prompt_updates
512 - for match in re.finditer(re.escape(update.target.text), prompt)
636 + match for update in prompt_updates for match in get_matches(update)
637 ]
638
639
@@ -547,45 +671,39 @@ def _apply_matches(
671 prev_end_idx = 0
672 next_idx_by_modality = defaultdict[str, int](lambda: 0)
673
550 - for (start_idx, end_idx), group in groupby(
551 - _resolve_matches(prompt, mm_matches),
552 - key=lambda x: (x.start_idx, x.end_idx),
553 - ):
554 - matches = tuple(group)
555 - assert len(matches) == 1
556 -
557 - for match in matches:
558 - modality = match.modality
674 + for match in _resolve_matches(prompt, mm_matches):
675 + modality = match.modality
676 +
677 + item_start_idx = next_idx_by_modality[modality]
678 + max_item_count = mm_item_counts.get(modality, 0)
679 + if item_start_idx >= max_item_count:
680 + continue
681 +
682 + start_idx = match.start_idx
683 + end_idx = match.end_idx
684 + origin = match._origin
685 + mode = origin.mode
686 +
687 + if mode == UpdateMode.INSERT:
688 + out_seqs.append(prompt[prev_end_idx:end_idx])
689 + num_inserts = max_item_count
690 + elif mode == UpdateMode.REPLACE:
691 + out_seqs.append(prompt[prev_end_idx:start_idx])
692 + num_inserts = max_item_count if start_idx == end_idx else 1
693 + else:
694 + assert_never(mode)
695
560 - item_idx = next_idx_by_modality[modality]
561 - if item_idx >= mm_item_counts.get(modality, 0):
562 - continue
696 + item_end_idx = min(item_start_idx + num_inserts, max_item_count)
697
564 - origin = match._origin
698 + for item_idx in range(item_start_idx, item_end_idx):
699 content = origin.get_content(item_idx)
566 - mode = origin.mode
567 -
568 - if mode == UpdateMode.INSERT:
569 - out_seqs.append(prompt[prev_end_idx:end_idx])
570 - num_inserts = mm_item_counts.get(modality, 0)
571 - elif mode == UpdateMode.REPLACE:
572 - out_seqs.append(prompt[prev_end_idx:start_idx])
573 - num_inserts = 1
574 - else:
575 - assert_never(mode)
576 -
577 - for _ in range(num_inserts):
578 - if item_idx >= mm_item_counts.get(modality, 0):
579 - continue
580 -
581 - if isinstance(prompt, str):
582 - out_seqs.append(content.full.text)
583 - else:
584 - out_seqs.append(content.full.token_ids)
700 + insert_seq = (content.full.text if isinstance(prompt, str) else
701 + content.full.token_ids)
702
586 - next_idx_by_modality[modality] += 1
703 + out_seqs.append(insert_seq)
704
588 - prev_end_idx = end_idx
705 + prev_end_idx = end_idx
706 + next_idx_by_modality[modality] += item_end_idx - item_start_idx
707
708 out_seqs.append(prompt[prev_end_idx:])
709