@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df76e5af26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[VLM] Simplify post-processing of replacement info (#12269)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 22, 2025 at 08:48 UTC df76e5af2635dee68bbace85da6c0257f6ee74bb
10 files changed +175 -208
tests/models/multimodal/processing/test_common.py
+1 -1
@@ -35,7 +35,7 @@ def _test_processing_correctness(
35 task="auto",
36 tokenizer=model_id,
37 tokenizer_mode="auto",
38 - trust_remote_code=True,
38 + trust_remote_code=model_info.trust_remote_code,
39 seed=0,
40 dtype="float16",
41 revision=None,
tests/models/registry.py
+2 -1
@@ -261,7 +261,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
261 trust_remote_code=True),
262 "Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
263 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
264 - "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
264 + "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
265 + trust_remote_code=True),
266 # [Encoder-decoder]
267 "MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
268 "WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
tests/multimodal/test_processing.py
+23 -19
@@ -7,12 +7,16 @@ import pytest
7
8 from vllm.config import ModelConfig
9 from vllm.multimodal import MULTIMODAL_REGISTRY
10 -from vllm.multimodal.processing import (PlaceholderInfo, PromptReplacement,
10 +# yapf conflicts with isort for this block
11 +# yapf: disable
12 +from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
13 + PromptReplacement,
14 find_mm_placeholders,
15 find_text_matches, find_token_matches,
16 iter_token_matches,
17 replace_text_matches,
18 replace_token_matches)
19 +# yapf: enable
20 from vllm.multimodal.profiling import MultiModalProfiler
21 from vllm.multimodal.utils import cached_get_tokenizer
22 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -433,19 +437,19 @@ def test_find_replace_tokens(
437 [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
438 {
439 "pattern_1": [
436 - PlaceholderInfo(
440 + PlaceholderFeaturesInfo(
441 modality="pattern_1",
442 item_idx=0,
443 start_idx=6,
440 - replacement=[32000, 32000],
444 + tokens=[32000, 32000],
445 ),
446 ],
447 "pattern_4": [
444 - PlaceholderInfo(
448 + PlaceholderFeaturesInfo(
449 modality="pattern_4",
450 item_idx=0,
451 start_idx=3,
448 - replacement=[32000],
452 + tokens=[32000],
453 ),
454 ],
455 }
@@ -455,25 +459,25 @@ def test_find_replace_tokens(
459 [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
460 {
461 "pattern_1": [
458 - PlaceholderInfo(
462 + PlaceholderFeaturesInfo(
463 modality="pattern_1",
464 item_idx=0,
465 start_idx=1,
462 - replacement=[32000, 32000],
466 + tokens=[32000, 32000],
467 ),
464 - PlaceholderInfo(
468 + PlaceholderFeaturesInfo(
469 modality="pattern_1",
470 item_idx=1,
471 start_idx=5,
468 - replacement=[32000, 32000],
472 + tokens=[32000, 32000],
473 ),
474 ],
475 "pattern_3": [
472 - PlaceholderInfo(
476 + PlaceholderFeaturesInfo(
477 modality="pattern_3",
478 item_idx=0,
479 start_idx=7,
476 - replacement=[1550, 918, 1550],
480 + tokens=[1550, 918, 1550],
481 ),
482 ],
483 # No match for pattern_4 as it has lower priority than pattern_1
@@ -483,33 +487,33 @@ def test_find_replace_tokens(
487 [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
488 {
489 "pattern_1": [
486 - PlaceholderInfo(
490 + PlaceholderFeaturesInfo(
491 modality="pattern_1",
492 item_idx=0,
493 start_idx=1,
490 - replacement=[32000, 32000],
494 + tokens=[32000, 32000],
495 ),
492 - PlaceholderInfo(
496 + PlaceholderFeaturesInfo(
497 modality="pattern_1",
498 item_idx=1,
499 start_idx=3,
496 - replacement=[32000, 32000],
500 + tokens=[32000, 32000],
501 ),
502 ],
503 "pattern_4": [
500 - PlaceholderInfo(
504 + PlaceholderFeaturesInfo(
505 modality="pattern_4",
506 item_idx=0,
507 start_idx=5,
504 - replacement=[32000],
508 + tokens=[32000],
509 ),
510 ],
511 "pattern_3": [
508 - PlaceholderInfo(
512 + PlaceholderFeaturesInfo(
513 modality="pattern_3",
514 item_idx=0,
515 start_idx=6,
512 - replacement=[1550, 918, 1550],
516 + tokens=[1550, 918, 1550],
517 ),
518 ],
519 }
vllm/model_executor/models/aria.py
+2 -8
@@ -342,13 +342,7 @@ class AriaProcessingInfo(BaseProcessingInfo):
342 return self.get_hf_config().vision_config
343
344 def get_hf_processor(self):
345 - processor = self.ctx.get_hf_processor(AriaProcessor)
346 -
347 - # Patch for https://github.com/huggingface/transformers/issues/35768
348 - processor.tokenizer.image_token = "<|img|>"
349 - processor.image_token = "<|img|>"
350 -
351 - return processor
345 + return self.ctx.get_hf_processor(AriaProcessor)
346
347 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
348 return {"image": None}
@@ -381,7 +375,7 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
375 }
376
377 hf_processor = self.info.get_hf_processor()
384 - image_token: str = hf_processor.image_token # type: ignore
378 + image_token: str = hf_processor.tokenizer.image_token # type: ignore
379
380 return ProcessorInputs(
381 prompt_text=image_token * num_images,
vllm/model_executor/models/blip2.py
+8 -25
@@ -14,12 +14,12 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15 from vllm.model_executor.sampling_metadata import SamplingMetadata
16 from vllm.multimodal import MULTIMODAL_REGISTRY
17 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18 - MultiModalInputs, MultiModalKwargs,
19 - NestedTensors, PlaceholderRange)
17 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
18 + NestedTensors)
19 from vllm.multimodal.parse import MultiModalDataItems
20 from vllm.multimodal.processing import (BaseMultiModalProcessor,
22 - BaseProcessingInfo, PromptReplacement)
21 + BaseProcessingInfo, PromptReplacement,
22 + PromptReplacementDetails)
23 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24 from vllm.sequence import IntermediateTensors
25
@@ -481,30 +481,13 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
481 PromptReplacement(
482 modality="image",
483 target="</s>",
484 - replacement="<image>" * num_image_tokens + "</s>",
484 + replacement=PromptReplacementDetails(
485 + full="<image>" * num_image_tokens + "</s>",
486 + features="<image>" * num_image_tokens,
487 + ),
488 )
489 ]
490
488 - def apply(
489 - self,
490 - prompt: Union[str, list[int]],
491 - mm_data: MultiModalDataDict,
492 - hf_processor_mm_kwargs: Mapping[str, object],
493 - ) -> MultiModalInputs:
494 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
495 -
496 - # Only <image> tokens should be considered as placeholders,
497 - # so we ignore the trailing bos_token
498 - result["mm_placeholders"] = {
499 - modality: [
500 - PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
501 - for p in ps
502 - ]
503 - for modality, ps in result["mm_placeholders"].items()
504 - }
505 -
506 - return result
507 -
491
492 @MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor,
493 info=Blip2ProcessingInfo,
vllm/model_executor/models/chameleon.py
+13 -29
@@ -28,12 +28,12 @@ from vllm.model_executor.model_loader.weight_utils import (
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.model_executor.utils import set_weight_attrs
30 from vllm.multimodal import MULTIMODAL_REGISTRY
31 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 - MultiModalInputs, MultiModalKwargs,
33 - NestedTensors, PlaceholderRange)
31 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
32 + NestedTensors)
33 from vllm.multimodal.parse import MultiModalDataItems
34 from vllm.multimodal.processing import (BaseMultiModalProcessor,
36 - BaseProcessingInfo, PromptReplacement)
35 + BaseProcessingInfo, PromptReplacement,
36 + PromptReplacementDetails)
37 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 from vllm.sequence import IntermediateTensors
39
@@ -141,39 +141,23 @@ class ChameleonMultiModalProcessor(
141 out_mm_kwargs: MultiModalKwargs,
142 ) -> list[PromptReplacement]:
143 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
144 + image_tokens = processor.image_token * self.info.get_num_image_tokens()
145
146 return [
147 PromptReplacement(
148 modality="image",
149 target="<image>",
149 - replacement="".join([
150 - processor.image_start_token,
151 - processor.image_token * self.info.get_num_image_tokens(),
152 - processor.image_end_token,
153 - ]),
150 + replacement=PromptReplacementDetails(
151 + full="".join([
152 + processor.image_start_token,
153 + image_tokens,
154 + processor.image_end_token,
155 + ]),
156 + features=image_tokens,
157 + ),
158 )
159 ]
160
157 - def apply(
158 - self,
159 - prompt: Union[str, list[int]],
160 - mm_data: MultiModalDataDict,
161 - hf_processor_mm_kwargs: Mapping[str, object],
162 - ) -> MultiModalInputs:
163 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
164 -
165 - # Only <image> tokens should be considered as placeholders,
166 - # so we ignore the image_start_token and image_end_token
167 - result["mm_placeholders"] = {
168 - modality: [
169 - PlaceholderRange(offset=p["offset"] + 1,
170 - length=p["length"] - 2) for p in ps
171 - ]
172 - for modality, ps in result["mm_placeholders"].items()
173 - }
174 -
175 - return result
176 -
161
162 class ChameleonLayerNorm(nn.LayerNorm):
163
vllm/model_executor/models/fuyu.py
+11 -27
@@ -16,7 +16,7 @@
16 """ PyTorch Fuyu model."""
17 import math
18 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19 - TypedDict, Union)
19 + TypedDict)
20
21 import torch
22 import torch.nn as nn
@@ -30,13 +30,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput
30 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 - MultiModalInputs, MultiModalKwargs,
35 - NestedTensors, PlaceholderRange)
33 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
34 + NestedTensors)
35 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
36 MultiModalDataItems)
37 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 - BaseProcessingInfo, PromptReplacement)
38 + BaseProcessingInfo, PromptReplacement,
39 + PromptReplacementDetails)
40 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41 from vllm.sequence import IntermediateTensors
42
@@ -215,9 +215,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
215 image_width=image_size.width,
216 image_height=image_size.height,
217 )
218 + image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
219 + [_NEWLINE_TOKEN_ID]) * nrows
220
219 - return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
220 - [bos_token_id])
221 + return PromptReplacementDetails(
222 + full=image_tokens + [bos_token_id],
223 + features=image_tokens,
224 + )
225
226 return [
227 PromptReplacement(
@@ -227,26 +231,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
231 )
232 ]
233
230 - def apply(
231 - self,
232 - prompt: Union[str, list[int]],
233 - mm_data: MultiModalDataDict,
234 - hf_processor_mm_kwargs: Mapping[str, object],
235 - ) -> MultiModalInputs:
236 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
237 -
238 - # Only |SPEAKER| (image) tokens should be considered as placeholders,
239 - # so we ignore the trailing bos_token_id
240 - result["mm_placeholders"] = {
241 - modality: [
242 - PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
243 - for p in ps
244 - ]
245 - for modality, ps in result["mm_placeholders"].items()
246 - }
247 -
248 - return result
249 -
234
235 @MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor,
236 info=FuyuProcessingInfo,
vllm/model_executor/models/phi3v.py
+17 -28
@@ -30,15 +30,19 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30 VocabParallelEmbedding)
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 - MultiModalInputs, MultiModalKwargs,
35 - NestedTensors, PlaceholderRange)
33 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
34 + NestedTensors)
35 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
36 ImageSize, MultiModalDataItems)
37 +# yapf conflicts with isort for this block
38 +# yapf: disable
39 from vllm.multimodal.processing import (BaseMultiModalProcessor,
40 BaseProcessingInfo,
41 BoundPromptReplacement,
41 - PlaceholderInfo, PromptReplacement)
42 + PlaceholderFeaturesInfo,
43 + PromptReplacement,
44 + PromptReplacementDetails)
45 +# yapf: enable
46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 from vllm.sequence import IntermediateTensors
48 from vllm.utils import is_list_of
@@ -437,7 +441,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
441 processor=hf_processor,
442 )
443
440 - return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
444 + image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
445 +
446 + return PromptReplacementDetails(
447 + full=image_tokens + [bos_token_id],
448 + features=image_tokens,
449 + )
450
451 num_images = mm_items.get_count("image", strict=False)
452
@@ -454,7 +463,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
463 token_ids: list[int],
464 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
465 mm_item_counts: Mapping[str, int],
457 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
466 + ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
467 token_ids, text, placeholders = super()._apply_prompt_replacements(
468 token_ids=token_ids,
469 mm_prompt_repls=mm_prompt_repls,
@@ -467,11 +476,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
476 token_ids = [token_ids[0], *token_ids[2:]]
477 placeholders = {
478 modality: [
470 - PlaceholderInfo(
479 + PlaceholderFeaturesInfo(
480 modality=p.modality,
481 item_idx=p.item_idx,
482 start_idx=p.start_idx - 1,
474 - replacement=p.replacement,
483 + tokens=p.tokens,
484 ) for p in ps
485 ]
486 for modality, ps in placeholders.items()
@@ -479,26 +488,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
488
489 return token_ids, text, placeholders
490
482 - def apply(
483 - self,
484 - prompt: Union[str, list[int]],
485 - mm_data: MultiModalDataDict,
486 - hf_processor_mm_kwargs: Mapping[str, object],
487 - ) -> MultiModalInputs:
488 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
489 -
490 - # Only <|image|> tokens should be considered as placeholders,
491 - # so we ignore the trailing bos_token_id
492 - result["mm_placeholders"] = {
493 - modality: [
494 - PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
495 - for p in ps
496 - ]
497 - for modality, ps in result["mm_placeholders"].items()
498 - }
499 -
500 - return result
501 -
491
492 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,
493 info=Phi3VProcessingInfo,
vllm/model_executor/models/qwen2_audio.py
+14 -29
@@ -36,13 +36,13 @@ from vllm.config import VllmConfig
36 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 -from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
40 - MultiModalInputs, MultiModalKwargs,
41 - NestedTensors, PlaceholderRange)
39 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40 + NestedTensors)
41 from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42 MultiModalDataParser)
43 from vllm.multimodal.processing import (BaseMultiModalProcessor,
45 - BaseProcessingInfo, PromptReplacement)
44 + BaseProcessingInfo, PromptReplacement,
45 + PromptReplacementDetails)
46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 from vllm.sequence import IntermediateTensors
48
@@ -216,11 +216,16 @@ class Qwen2AudioMultiModalProcessor(
216 f"The audio {audio} (len={len(audio)}) is too short "
217 "to be represented inside the model")
218
219 - return "".join([
220 - audio_bos_token,
221 - audio_token * num_placeholders,
222 - audio_eos_token,
223 - ])
219 + audio_tokens = audio_token * num_placeholders
220 +
221 + return PromptReplacementDetails(
222 + full="".join([
223 + audio_bos_token,
224 + audio_tokens,
225 + audio_eos_token,
226 + ]),
227 + features=audio_tokens,
228 + )
229
230 return [
231 PromptReplacement(
@@ -240,26 +245,6 @@ class Qwen2AudioMultiModalProcessor(
245 # tokens than the number of audio items)
246 return not hasattr(self.info.get_hf_processor(), "audio_token")
247
243 - def apply(
244 - self,
245 - prompt: Union[str, list[int]],
246 - mm_data: MultiModalDataDict,
247 - hf_processor_mm_kwargs: Mapping[str, object],
248 - ) -> MultiModalInputs:
249 - result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
250 -
251 - # Only <|AUDIO|> tokens should be considered as placeholders,
252 - # so we ignore the audio_bos_token and audio_eos_token
253 - result["mm_placeholders"] = {
254 - modality: [
255 - PlaceholderRange(offset=p["offset"] + 1,
256 - length=p["length"] - 2) for p in ps
257 - ]
258 - for modality, ps in result["mm_placeholders"].items()
259 - }
260 -
261 - return result
262 -
248
249 @MULTIMODAL_REGISTRY.register_processor(
250 Qwen2AudioMultiModalProcessor,
vllm/multimodal/processing.py
+84 -41
@@ -1,7 +1,8 @@
1 import re
2 from abc import ABC, abstractmethod
3 from collections import defaultdict
4 -from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4 +from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
5 + Sequence)
6 from dataclasses import dataclass, field
7 from functools import lru_cache
8 from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
@@ -31,6 +32,24 @@ _S = TypeVar("_S", str, list[int])
32 _PromptSeq = Union[str, list[int]]
33
34
35 +@dataclass
36 +class PromptReplacementDetails:
37 + full: _PromptSeq
38 + """The full replacement."""
39 +
40 + features: _PromptSeq
41 + """
42 + The part of the replacement that corresponds to placeholder feature tokens.
43 + """
44 +
45 + @staticmethod
46 + def from_seq(seq: _PromptSeq):
47 + return PromptReplacementDetails(full=seq, features=seq)
48 +
49 +
50 +_PromptRepl = Union[_PromptSeq, PromptReplacementDetails]
51 +
52 +
53 @dataclass
54 class PromptReplacement:
55 """
@@ -43,8 +62,8 @@ class PromptReplacement:
62 target: _PromptSeq
63 """The token sequence (or text) to find and replace."""
64
46 - replacement: Union[Callable[[int], _PromptSeq],
47 - _PromptSeq] = field(repr=False)
65 + replacement: Union[Callable[[int], _PromptRepl],
66 + _PromptRepl] = field(repr=False)
67 """
68 Given the index of the processed item within :attr:`modality`,
69 output the replacement token sequence (or text).
@@ -112,6 +131,14 @@ class _BoundPromptSequence:
131 _text: Optional[str]
132 _token_ids: Optional[list[int]]
133
134 + @staticmethod
135 + def from_seq(tokenizer: AnyTokenizer, seq: _PromptSeq):
136 + return _BoundPromptSequence(
137 + tokenizer=tokenizer,
138 + _text=seq if isinstance(seq, str) else None,
139 + _token_ids=seq if isinstance(seq, list) else None,
140 + )
141 +
142 def __post_init__(self) -> None:
143 if self._text is None and self._token_ids is None:
144 raise ValueError("At least one of 'text' and 'token_ids' must be "
@@ -134,6 +161,12 @@ class _BoundPromptSequence:
161 return self._token_ids
162
163
164 +@dataclass
165 +class _BoundPromptReplacementGroup:
166 + full: _BoundPromptSequence
167 + features: _BoundPromptSequence
168 +
169 +
170 @dataclass
171 class BoundPromptReplacement:
172 """
@@ -145,24 +178,18 @@ class BoundPromptReplacement:
178 modality: str
179
180 _target: _PromptSeq
148 - _replacement: Union[Callable[[int], _PromptSeq],
149 - _PromptSeq] = field(repr=False)
181 + _replacement: Union[Callable[[int], _PromptRepl],
182 + _PromptRepl] = field(repr=False)
183
184 def __post_init__(self) -> None:
152 - self._replacement_cache = dict[int, _BoundPromptSequence]()
185 + self._replacement_cache = dict[int, _BoundPromptReplacementGroup]()
186
187 @property
188 def target(self) -> _BoundPromptSequence:
189 """The token sequence (or text) to find and replace."""
157 - target = self._target
190 + return _BoundPromptSequence.from_seq(self.tokenizer, self._target)
191
159 - return _BoundPromptSequence(
160 - tokenizer=self.tokenizer,
161 - _text=target if isinstance(target, str) else None,
162 - _token_ids=target if isinstance(target, list) else None,
163 - )
164 -
165 - def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
192 + def get_replacement(self, item_idx: int) -> _BoundPromptReplacementGroup:
193 """
194 Given the index of the processed item within :attr:`modality`,
195 output the replacement token sequence (or text).
@@ -177,10 +204,16 @@ class BoundPromptReplacement:
204 else:
205 cache_key = None
206
180 - bound_replacement = _BoundPromptSequence(
181 - tokenizer=self.tokenizer,
182 - _text=replacement if isinstance(replacement, str) else None,
183 - _token_ids=replacement if isinstance(replacement, list) else None,
207 + if not isinstance(replacement, PromptReplacementDetails):
208 + replacement = PromptReplacementDetails.from_seq(replacement)
209 +
210 + bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
211 + replacement.full)
212 + bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
213 + replacement.features)
214 + bound_replacement = _BoundPromptReplacementGroup(
215 + full=bound_full,
216 + features=bound_features,
217 )
218
219 if cache_key is not None:
@@ -197,7 +230,7 @@ class _TokenMatch(NamedTuple):
230 def iter_token_matches(
231 token_ids: list[int],
232 match_ids: list[int],
200 -) -> Iterable[_TokenMatch]:
233 +) -> Generator[_TokenMatch]:
234 """
235 Yield each occurrence of :code:`match_ids` in :code:`token_ids`.
236
@@ -272,15 +305,15 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
305
306
307 @dataclass
275 -class PlaceholderInfo:
308 +class PlaceholderFeaturesInfo:
309 modality: str
310 item_idx: int
311 start_idx: int
279 - replacement: list[int]
312 + tokens: list[int]
313
314 @property
315 def length(self) -> int:
283 - return len(self.replacement)
316 + return len(self.tokens)
317
318 def to_range(self) -> PlaceholderRange:
319 return PlaceholderRange(
@@ -362,10 +395,10 @@ def _replace_matches(
395 replacement = repl_info.get_replacement(item_idx)
396
397 if isinstance(prompt, str):
365 - repl_seq = replacement.text
398 + repl_seq = replacement.full.text
399 out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
400 else:
368 - repl_seq = replacement.token_ids
401 + repl_seq = replacement.full.token_ids
402 out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
403
404 prev_end_idx = end_idx
@@ -408,7 +441,7 @@ def _iter_placeholders(
441 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
442 prompt: list[int],
443 mm_item_counts: Mapping[str, int],
411 -) -> Iterable[PlaceholderInfo]:
444 +) -> Iterable[PlaceholderFeaturesInfo]:
445 """
446 Yield each set of placeholder tokens found in :code:`prompt`.
447
@@ -432,23 +465,33 @@ def _iter_placeholders(
465
466 for repl_info in modality_repls:
467 replacement = repl_info.get_replacement(item_idx)
435 - repl_tokens = replacement.token_ids
436 - repl_len = len(repl_tokens)
437 - end_idx = start_idx + repl_len
468 + repl_tokens_full = replacement.full.token_ids
469 + repl_len_full = len(repl_tokens_full)
470 + end_idx_full = start_idx + repl_len_full
471
439 - if repl_len == 0 or end_idx > prompt_len:
472 + if repl_len_full == 0 or end_idx_full > prompt_len:
473 continue
474
442 - if prompt[start_idx:end_idx] == repl_tokens:
443 - yield PlaceholderInfo(
444 - modality=modality,
445 - item_idx=item_idx,
446 - start_idx=start_idx,
447 - replacement=repl_tokens,
448 - )
475 + if prompt[start_idx:end_idx_full] == repl_tokens_full:
476 + repl_tokens_feat = replacement.features.token_ids
477 +
478 + try:
479 + match = next(
480 + iter_token_matches(repl_tokens_full,
481 + repl_tokens_feat))
482 + yield PlaceholderFeaturesInfo(
483 + modality=modality,
484 + item_idx=item_idx,
485 + start_idx=start_idx + match.start_idx,
486 + tokens=repl_tokens_feat,
487 + )
488 + except StopIteration:
489 + raise AssertionError(
490 + f"{repl_tokens_feat=} should be a "
491 + f"subsequence of {repl_tokens_full=}") from None
492
493 # Exclude overlapping matches
451 - start_idx = end_idx
494 + start_idx = end_idx_full
495 item_idx_by_modality[modality] += 1
496 found = True
497 break
@@ -464,7 +507,7 @@ def find_mm_placeholders(
507 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
508 prompt: list[int],
509 mm_item_counts: Mapping[str, int],
467 -) -> Mapping[str, list[PlaceholderInfo]]:
510 +) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
511 it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
512 return dict(full_groupby_modality(it))
513
@@ -679,7 +722,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
722 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
723 new_token_ids: list[int],
724 mm_item_counts: Mapping[str, int],
682 - ) -> Mapping[str, list[PlaceholderInfo]]:
725 + ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
726 return find_mm_placeholders(mm_prompt_repls, new_token_ids,
727 mm_item_counts)
728
@@ -948,7 +991,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
991 token_ids: list[int],
992 mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
993 mm_item_counts: Mapping[str, int],
951 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
994 + ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
995 tokenizer = self.info.get_tokenizer()
996
997 mm_token_matches = {
@@ -1037,7 +1080,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1080
1081 def _validate_mm_placeholders(
1082 self,
1040 - mm_placeholders: Mapping[str, list[PlaceholderInfo]],
1083 + mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1084 mm_item_counts: Mapping[str, int],
1085 *,
1086 allow_missing: bool = False,