A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[VLM] Simplify post-processing of replacement info (#12269)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 22, 2025 at 08:48 UTC
df76e5af2635dee68bbace85da6c0257f6ee74bb
10 files changed
+175
-208
tests/models/multimodal/processing/test_common.py
+1
-1
@@ -35,7 +35,7 @@ def _test_processing_correctness(
35
task="auto",
36
tokenizer=model_id,
37
tokenizer_mode="auto",
38
- trust_remote_code=True,
38
+ trust_remote_code=model_info.trust_remote_code,
39
seed=0,
40
dtype="float16",
41
revision=None,
tests/models/registry.py
+2
-1
@@ -261,7 +261,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
261
trust_remote_code=True),
262
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
263
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
264
- "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3"),
264
+ "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
265
+ trust_remote_code=True),
266
# [Encoder-decoder]
267
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
268
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
tests/multimodal/test_processing.py
+23
-19
@@ -7,12 +7,16 @@ import pytest
7
8
from vllm.config import ModelConfig
9
from vllm.multimodal import MULTIMODAL_REGISTRY
10
-from vllm.multimodal.processing import (PlaceholderInfo, PromptReplacement,
10
+# yapf conflicts with isort for this block
11
+# yapf: disable
12
+from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
13
+ PromptReplacement,
14
find_mm_placeholders,
15
find_text_matches, find_token_matches,
16
iter_token_matches,
17
replace_text_matches,
18
replace_token_matches)
19
+# yapf: enable
20
from vllm.multimodal.profiling import MultiModalProfiler
21
from vllm.multimodal.utils import cached_get_tokenizer
22
from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -433,19 +437,19 @@ def test_find_replace_tokens(
437
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
438
{
439
"pattern_1": [
436
- PlaceholderInfo(
440
+ PlaceholderFeaturesInfo(
441
modality="pattern_1",
442
item_idx=0,
443
start_idx=6,
440
- replacement=[32000, 32000],
444
+ tokens=[32000, 32000],
445
),
446
],
447
"pattern_4": [
444
- PlaceholderInfo(
448
+ PlaceholderFeaturesInfo(
449
modality="pattern_4",
450
item_idx=0,
451
start_idx=3,
448
- replacement=[32000],
452
+ tokens=[32000],
453
),
454
],
455
}
@@ -455,25 +459,25 @@ def test_find_replace_tokens(
459
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
460
{
461
"pattern_1": [
458
- PlaceholderInfo(
462
+ PlaceholderFeaturesInfo(
463
modality="pattern_1",
464
item_idx=0,
465
start_idx=1,
462
- replacement=[32000, 32000],
466
+ tokens=[32000, 32000],
467
),
464
- PlaceholderInfo(
468
+ PlaceholderFeaturesInfo(
469
modality="pattern_1",
470
item_idx=1,
471
start_idx=5,
468
- replacement=[32000, 32000],
472
+ tokens=[32000, 32000],
473
),
474
],
475
"pattern_3": [
472
- PlaceholderInfo(
476
+ PlaceholderFeaturesInfo(
477
modality="pattern_3",
478
item_idx=0,
479
start_idx=7,
476
- replacement=[1550, 918, 1550],
480
+ tokens=[1550, 918, 1550],
481
),
482
],
483
# No match for pattern_4 as it has lower priority than pattern_1
@@ -483,33 +487,33 @@ def test_find_replace_tokens(
487
[1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
488
{
489
"pattern_1": [
486
- PlaceholderInfo(
490
+ PlaceholderFeaturesInfo(
491
modality="pattern_1",
492
item_idx=0,
493
start_idx=1,
490
- replacement=[32000, 32000],
494
+ tokens=[32000, 32000],
495
),
492
- PlaceholderInfo(
496
+ PlaceholderFeaturesInfo(
497
modality="pattern_1",
498
item_idx=1,
499
start_idx=3,
496
- replacement=[32000, 32000],
500
+ tokens=[32000, 32000],
501
),
502
],
503
"pattern_4": [
500
- PlaceholderInfo(
504
+ PlaceholderFeaturesInfo(
505
modality="pattern_4",
506
item_idx=0,
507
start_idx=5,
504
- replacement=[32000],
508
+ tokens=[32000],
509
),
510
],
511
"pattern_3": [
508
- PlaceholderInfo(
512
+ PlaceholderFeaturesInfo(
513
modality="pattern_3",
514
item_idx=0,
515
start_idx=6,
512
- replacement=[1550, 918, 1550],
516
+ tokens=[1550, 918, 1550],
517
),
518
],
519
}
vllm/model_executor/models/aria.py
+2
-8
@@ -342,13 +342,7 @@ class AriaProcessingInfo(BaseProcessingInfo):
342
return self.get_hf_config().vision_config
343
344
def get_hf_processor(self):
345
- processor = self.ctx.get_hf_processor(AriaProcessor)
346
-
347
- # Patch for https://github.com/huggingface/transformers/issues/35768
348
- processor.tokenizer.image_token = "<|img|>"
349
- processor.image_token = "<|img|>"
350
-
351
- return processor
345
+ return self.ctx.get_hf_processor(AriaProcessor)
346
347
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
348
return {"image": None}
@@ -381,7 +375,7 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
375
}
376
377
hf_processor = self.info.get_hf_processor()
384
- image_token: str = hf_processor.image_token # type: ignore
378
+ image_token: str = hf_processor.tokenizer.image_token # type: ignore
379
380
return ProcessorInputs(
381
prompt_text=image_token * num_images,
vllm/model_executor/models/blip2.py
+8
-25
@@ -14,12 +14,12 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15
from vllm.model_executor.sampling_metadata import SamplingMetadata
16
from vllm.multimodal import MULTIMODAL_REGISTRY
17
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18
- MultiModalInputs, MultiModalKwargs,
19
- NestedTensors, PlaceholderRange)
17
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
18
+ NestedTensors)
19
from vllm.multimodal.parse import MultiModalDataItems
20
from vllm.multimodal.processing import (BaseMultiModalProcessor,
22
- BaseProcessingInfo, PromptReplacement)
21
+ BaseProcessingInfo, PromptReplacement,
22
+ PromptReplacementDetails)
23
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24
from vllm.sequence import IntermediateTensors
25
@@ -481,30 +481,13 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
481
PromptReplacement(
482
modality="image",
483
target="</s>",
484
- replacement="<image>" * num_image_tokens + "</s>",
484
+ replacement=PromptReplacementDetails(
485
+ full="<image>" * num_image_tokens + "</s>",
486
+ features="<image>" * num_image_tokens,
487
+ ),
488
)
489
]
490
488
- def apply(
489
- self,
490
- prompt: Union[str, list[int]],
491
- mm_data: MultiModalDataDict,
492
- hf_processor_mm_kwargs: Mapping[str, object],
493
- ) -> MultiModalInputs:
494
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
495
-
496
- # Only <image> tokens should be considered as placeholders,
497
- # so we ignore the trailing bos_token
498
- result["mm_placeholders"] = {
499
- modality: [
500
- PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
501
- for p in ps
502
- ]
503
- for modality, ps in result["mm_placeholders"].items()
504
- }
505
-
506
- return result
507
-
491
492
@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor,
493
info=Blip2ProcessingInfo,
vllm/model_executor/models/chameleon.py
+13
-29
@@ -28,12 +28,12 @@ from vllm.model_executor.model_loader.weight_utils import (
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.model_executor.utils import set_weight_attrs
30
from vllm.multimodal import MULTIMODAL_REGISTRY
31
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32
- MultiModalInputs, MultiModalKwargs,
33
- NestedTensors, PlaceholderRange)
31
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
32
+ NestedTensors)
33
from vllm.multimodal.parse import MultiModalDataItems
34
from vllm.multimodal.processing import (BaseMultiModalProcessor,
36
- BaseProcessingInfo, PromptReplacement)
35
+ BaseProcessingInfo, PromptReplacement,
36
+ PromptReplacementDetails)
37
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38
from vllm.sequence import IntermediateTensors
39
@@ -141,39 +141,23 @@ class ChameleonMultiModalProcessor(
141
out_mm_kwargs: MultiModalKwargs,
142
) -> list[PromptReplacement]:
143
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
144
+ image_tokens = processor.image_token * self.info.get_num_image_tokens()
145
146
return [
147
PromptReplacement(
148
modality="image",
149
target="<image>",
149
- replacement="".join([
150
- processor.image_start_token,
151
- processor.image_token * self.info.get_num_image_tokens(),
152
- processor.image_end_token,
153
- ]),
150
+ replacement=PromptReplacementDetails(
151
+ full="".join([
152
+ processor.image_start_token,
153
+ image_tokens,
154
+ processor.image_end_token,
155
+ ]),
156
+ features=image_tokens,
157
+ ),
158
)
159
]
160
157
- def apply(
158
- self,
159
- prompt: Union[str, list[int]],
160
- mm_data: MultiModalDataDict,
161
- hf_processor_mm_kwargs: Mapping[str, object],
162
- ) -> MultiModalInputs:
163
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
164
-
165
- # Only <image> tokens should be considered as placeholders,
166
- # so we ignore the image_start_token and image_end_token
167
- result["mm_placeholders"] = {
168
- modality: [
169
- PlaceholderRange(offset=p["offset"] + 1,
170
- length=p["length"] - 2) for p in ps
171
- ]
172
- for modality, ps in result["mm_placeholders"].items()
173
- }
174
-
175
- return result
176
-
161
162
class ChameleonLayerNorm(nn.LayerNorm):
163
vllm/model_executor/models/fuyu.py
+11
-27
@@ -16,7 +16,7 @@
16
""" PyTorch Fuyu model."""
17
import math
18
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19
- TypedDict, Union)
19
+ TypedDict)
20
21
import torch
22
import torch.nn as nn
@@ -30,13 +30,13 @@ from vllm.model_executor.layers.sampler import SamplerOutput
30
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
- MultiModalInputs, MultiModalKwargs,
35
- NestedTensors, PlaceholderRange)
33
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
34
+ NestedTensors)
35
from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
36
MultiModalDataItems)
37
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
- BaseProcessingInfo, PromptReplacement)
38
+ BaseProcessingInfo, PromptReplacement,
39
+ PromptReplacementDetails)
40
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41
from vllm.sequence import IntermediateTensors
42
@@ -215,9 +215,13 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
215
image_width=image_size.width,
216
image_height=image_size.height,
217
)
218
+ image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
219
+ [_NEWLINE_TOKEN_ID]) * nrows
220
219
- return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
220
- [bos_token_id])
221
+ return PromptReplacementDetails(
222
+ full=image_tokens + [bos_token_id],
223
+ features=image_tokens,
224
+ )
225
226
return [
227
PromptReplacement(
@@ -227,26 +231,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
231
)
232
]
233
230
- def apply(
231
- self,
232
- prompt: Union[str, list[int]],
233
- mm_data: MultiModalDataDict,
234
- hf_processor_mm_kwargs: Mapping[str, object],
235
- ) -> MultiModalInputs:
236
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
237
-
238
- # Only |SPEAKER| (image) tokens should be considered as placeholders,
239
- # so we ignore the trailing bos_token_id
240
- result["mm_placeholders"] = {
241
- modality: [
242
- PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
243
- for p in ps
244
- ]
245
- for modality, ps in result["mm_placeholders"].items()
246
- }
247
-
248
- return result
249
-
234
235
@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor,
236
info=FuyuProcessingInfo,
vllm/model_executor/models/phi3v.py
+17
-28
@@ -30,15 +30,19 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
30
VocabParallelEmbedding)
31
from vllm.model_executor.sampling_metadata import SamplingMetadata
32
from vllm.multimodal import MULTIMODAL_REGISTRY
33
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
- MultiModalInputs, MultiModalKwargs,
35
- NestedTensors, PlaceholderRange)
33
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
34
+ NestedTensors)
35
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
36
ImageSize, MultiModalDataItems)
37
+# yapf conflicts with isort for this block
38
+# yapf: disable
39
from vllm.multimodal.processing import (BaseMultiModalProcessor,
40
BaseProcessingInfo,
41
BoundPromptReplacement,
41
- PlaceholderInfo, PromptReplacement)
42
+ PlaceholderFeaturesInfo,
43
+ PromptReplacement,
44
+ PromptReplacementDetails)
45
+# yapf: enable
46
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47
from vllm.sequence import IntermediateTensors
48
from vllm.utils import is_list_of
@@ -437,7 +441,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
441
processor=hf_processor,
442
)
443
440
- return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
444
+ image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
445
+
446
+ return PromptReplacementDetails(
447
+ full=image_tokens + [bos_token_id],
448
+ features=image_tokens,
449
+ )
450
451
num_images = mm_items.get_count("image", strict=False)
452
@@ -454,7 +463,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
463
token_ids: list[int],
464
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
465
mm_item_counts: Mapping[str, int],
457
- ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
466
+ ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
467
token_ids, text, placeholders = super()._apply_prompt_replacements(
468
token_ids=token_ids,
469
mm_prompt_repls=mm_prompt_repls,
@@ -467,11 +476,11 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
476
token_ids = [token_ids[0], *token_ids[2:]]
477
placeholders = {
478
modality: [
470
- PlaceholderInfo(
479
+ PlaceholderFeaturesInfo(
480
modality=p.modality,
481
item_idx=p.item_idx,
482
start_idx=p.start_idx - 1,
474
- replacement=p.replacement,
483
+ tokens=p.tokens,
484
) for p in ps
485
]
486
for modality, ps in placeholders.items()
@@ -479,26 +488,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
488
489
return token_ids, text, placeholders
490
482
- def apply(
483
- self,
484
- prompt: Union[str, list[int]],
485
- mm_data: MultiModalDataDict,
486
- hf_processor_mm_kwargs: Mapping[str, object],
487
- ) -> MultiModalInputs:
488
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
489
-
490
- # Only <|image|> tokens should be considered as placeholders,
491
- # so we ignore the trailing bos_token_id
492
- result["mm_placeholders"] = {
493
- modality: [
494
- PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
495
- for p in ps
496
- ]
497
- for modality, ps in result["mm_placeholders"].items()
498
- }
499
-
500
- return result
501
-
491
492
@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,
493
info=Phi3VProcessingInfo,
vllm/model_executor/models/qwen2_audio.py
+14
-29
@@ -36,13 +36,13 @@ from vllm.config import VllmConfig
36
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
40
- MultiModalInputs, MultiModalKwargs,
41
- NestedTensors, PlaceholderRange)
39
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40
+ NestedTensors)
41
from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42
MultiModalDataParser)
43
from vllm.multimodal.processing import (BaseMultiModalProcessor,
45
- BaseProcessingInfo, PromptReplacement)
44
+ BaseProcessingInfo, PromptReplacement,
45
+ PromptReplacementDetails)
46
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47
from vllm.sequence import IntermediateTensors
48
@@ -216,11 +216,16 @@ class Qwen2AudioMultiModalProcessor(
216
f"The audio {audio} (len={len(audio)}) is too short "
217
"to be represented inside the model")
218
219
- return "".join([
220
- audio_bos_token,
221
- audio_token * num_placeholders,
222
- audio_eos_token,
223
- ])
219
+ audio_tokens = audio_token * num_placeholders
220
+
221
+ return PromptReplacementDetails(
222
+ full="".join([
223
+ audio_bos_token,
224
+ audio_tokens,
225
+ audio_eos_token,
226
+ ]),
227
+ features=audio_tokens,
228
+ )
229
230
return [
231
PromptReplacement(
@@ -240,26 +245,6 @@ class Qwen2AudioMultiModalProcessor(
245
# tokens than the number of audio items)
246
return not hasattr(self.info.get_hf_processor(), "audio_token")
247
243
- def apply(
244
- self,
245
- prompt: Union[str, list[int]],
246
- mm_data: MultiModalDataDict,
247
- hf_processor_mm_kwargs: Mapping[str, object],
248
- ) -> MultiModalInputs:
249
- result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
250
-
251
- # Only <|AUDIO|> tokens should be considered as placeholders,
252
- # so we ignore the audio_bos_token and audio_eos_token
253
- result["mm_placeholders"] = {
254
- modality: [
255
- PlaceholderRange(offset=p["offset"] + 1,
256
- length=p["length"] - 2) for p in ps
257
- ]
258
- for modality, ps in result["mm_placeholders"].items()
259
- }
260
-
261
- return result
262
-
248
249
@MULTIMODAL_REGISTRY.register_processor(
250
Qwen2AudioMultiModalProcessor,
vllm/multimodal/processing.py
+84
-41
@@ -1,7 +1,8 @@
1
import re
2
from abc import ABC, abstractmethod
3
from collections import defaultdict
4
-from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4
+from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
5
+ Sequence)
6
from dataclasses import dataclass, field
7
from functools import lru_cache
8
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
@@ -31,6 +32,24 @@ _S = TypeVar("_S", str, list[int])
32
_PromptSeq = Union[str, list[int]]
33
34
35
+@dataclass
36
+class PromptReplacementDetails:
37
+ full: _PromptSeq
38
+ """The full replacement."""
39
+
40
+ features: _PromptSeq
41
+ """
42
+ The part of the replacement that corresponds to placeholder feature tokens.
43
+ """
44
+
45
+ @staticmethod
46
+ def from_seq(seq: _PromptSeq):
47
+ return PromptReplacementDetails(full=seq, features=seq)
48
+
49
+
50
+_PromptRepl = Union[_PromptSeq, PromptReplacementDetails]
51
+
52
+
53
@dataclass
54
class PromptReplacement:
55
"""
@@ -43,8 +62,8 @@ class PromptReplacement:
62
target: _PromptSeq
63
"""The token sequence (or text) to find and replace."""
64
46
- replacement: Union[Callable[[int], _PromptSeq],
47
- _PromptSeq] = field(repr=False)
65
+ replacement: Union[Callable[[int], _PromptRepl],
66
+ _PromptRepl] = field(repr=False)
67
"""
68
Given the index of the processed item within :attr:`modality`,
69
output the replacement token sequence (or text).
@@ -112,6 +131,14 @@ class _BoundPromptSequence:
131
_text: Optional[str]
132
_token_ids: Optional[list[int]]
133
134
+ @staticmethod
135
+ def from_seq(tokenizer: AnyTokenizer, seq: _PromptSeq):
136
+ return _BoundPromptSequence(
137
+ tokenizer=tokenizer,
138
+ _text=seq if isinstance(seq, str) else None,
139
+ _token_ids=seq if isinstance(seq, list) else None,
140
+ )
141
+
142
def __post_init__(self) -> None:
143
if self._text is None and self._token_ids is None:
144
raise ValueError("At least one of 'text' and 'token_ids' must be "
@@ -134,6 +161,12 @@ class _BoundPromptSequence:
161
return self._token_ids
162
163
164
+@dataclass
165
+class _BoundPromptReplacementGroup:
166
+ full: _BoundPromptSequence
167
+ features: _BoundPromptSequence
168
+
169
+
170
@dataclass
171
class BoundPromptReplacement:
172
"""
@@ -145,24 +178,18 @@ class BoundPromptReplacement:
178
modality: str
179
180
_target: _PromptSeq
148
- _replacement: Union[Callable[[int], _PromptSeq],
149
- _PromptSeq] = field(repr=False)
181
+ _replacement: Union[Callable[[int], _PromptRepl],
182
+ _PromptRepl] = field(repr=False)
183
184
def __post_init__(self) -> None:
152
- self._replacement_cache = dict[int, _BoundPromptSequence]()
185
+ self._replacement_cache = dict[int, _BoundPromptReplacementGroup]()
186
187
@property
188
def target(self) -> _BoundPromptSequence:
189
"""The token sequence (or text) to find and replace."""
157
- target = self._target
190
+ return _BoundPromptSequence.from_seq(self.tokenizer, self._target)
191
159
- return _BoundPromptSequence(
160
- tokenizer=self.tokenizer,
161
- _text=target if isinstance(target, str) else None,
162
- _token_ids=target if isinstance(target, list) else None,
163
- )
164
-
165
- def get_replacement(self, item_idx: int) -> _BoundPromptSequence:
192
+ def get_replacement(self, item_idx: int) -> _BoundPromptReplacementGroup:
193
"""
194
Given the index of the processed item within :attr:`modality`,
195
output the replacement token sequence (or text).
@@ -177,10 +204,16 @@ class BoundPromptReplacement:
204
else:
205
cache_key = None
206
180
- bound_replacement = _BoundPromptSequence(
181
- tokenizer=self.tokenizer,
182
- _text=replacement if isinstance(replacement, str) else None,
183
- _token_ids=replacement if isinstance(replacement, list) else None,
207
+ if not isinstance(replacement, PromptReplacementDetails):
208
+ replacement = PromptReplacementDetails.from_seq(replacement)
209
+
210
+ bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
211
+ replacement.full)
212
+ bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
213
+ replacement.features)
214
+ bound_replacement = _BoundPromptReplacementGroup(
215
+ full=bound_full,
216
+ features=bound_features,
217
)
218
219
if cache_key is not None:
@@ -197,7 +230,7 @@ class _TokenMatch(NamedTuple):
230
def iter_token_matches(
231
token_ids: list[int],
232
match_ids: list[int],
200
-) -> Iterable[_TokenMatch]:
233
+) -> Generator[_TokenMatch]:
234
"""
235
Yield each occurrence of :code:`match_ids` in :code:`token_ids`.
236
@@ -272,15 +305,15 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
305
306
307
@dataclass
275
-class PlaceholderInfo:
308
+class PlaceholderFeaturesInfo:
309
modality: str
310
item_idx: int
311
start_idx: int
279
- replacement: list[int]
312
+ tokens: list[int]
313
314
@property
315
def length(self) -> int:
283
- return len(self.replacement)
316
+ return len(self.tokens)
317
318
def to_range(self) -> PlaceholderRange:
319
return PlaceholderRange(
@@ -362,10 +395,10 @@ def _replace_matches(
395
replacement = repl_info.get_replacement(item_idx)
396
397
if isinstance(prompt, str):
365
- repl_seq = replacement.text
398
+ repl_seq = replacement.full.text
399
out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
400
else:
368
- repl_seq = replacement.token_ids
401
+ repl_seq = replacement.full.token_ids
402
out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
403
404
prev_end_idx = end_idx
@@ -408,7 +441,7 @@ def _iter_placeholders(
441
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
442
prompt: list[int],
443
mm_item_counts: Mapping[str, int],
411
-) -> Iterable[PlaceholderInfo]:
444
+) -> Iterable[PlaceholderFeaturesInfo]:
445
"""
446
Yield each set of placeholder tokens found in :code:`prompt`.
447
@@ -432,23 +465,33 @@ def _iter_placeholders(
465
466
for repl_info in modality_repls:
467
replacement = repl_info.get_replacement(item_idx)
435
- repl_tokens = replacement.token_ids
436
- repl_len = len(repl_tokens)
437
- end_idx = start_idx + repl_len
468
+ repl_tokens_full = replacement.full.token_ids
469
+ repl_len_full = len(repl_tokens_full)
470
+ end_idx_full = start_idx + repl_len_full
471
439
- if repl_len == 0 or end_idx > prompt_len:
472
+ if repl_len_full == 0 or end_idx_full > prompt_len:
473
continue
474
442
- if prompt[start_idx:end_idx] == repl_tokens:
443
- yield PlaceholderInfo(
444
- modality=modality,
445
- item_idx=item_idx,
446
- start_idx=start_idx,
447
- replacement=repl_tokens,
448
- )
475
+ if prompt[start_idx:end_idx_full] == repl_tokens_full:
476
+ repl_tokens_feat = replacement.features.token_ids
477
+
478
+ try:
479
+ match = next(
480
+ iter_token_matches(repl_tokens_full,
481
+ repl_tokens_feat))
482
+ yield PlaceholderFeaturesInfo(
483
+ modality=modality,
484
+ item_idx=item_idx,
485
+ start_idx=start_idx + match.start_idx,
486
+ tokens=repl_tokens_feat,
487
+ )
488
+ except StopIteration:
489
+ raise AssertionError(
490
+ f"{repl_tokens_feat=} should be a "
491
+ f"subsequence of {repl_tokens_full=}") from None
492
493
# Exclude overlapping matches
451
- start_idx = end_idx
494
+ start_idx = end_idx_full
495
item_idx_by_modality[modality] += 1
496
found = True
497
break
@@ -464,7 +507,7 @@ def find_mm_placeholders(
507
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
508
prompt: list[int],
509
mm_item_counts: Mapping[str, int],
467
-) -> Mapping[str, list[PlaceholderInfo]]:
510
+) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
511
it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
512
return dict(full_groupby_modality(it))
513
@@ -679,7 +722,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
722
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
723
new_token_ids: list[int],
724
mm_item_counts: Mapping[str, int],
682
- ) -> Mapping[str, list[PlaceholderInfo]]:
725
+ ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
726
return find_mm_placeholders(mm_prompt_repls, new_token_ids,
727
mm_item_counts)
728
@@ -948,7 +991,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
991
token_ids: list[int],
992
mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
993
mm_item_counts: Mapping[str, int],
951
- ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
994
+ ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
995
tokenizer = self.info.get_tokenizer()
996
997
mm_token_matches = {
@@ -1037,7 +1080,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1080
1081
def _validate_mm_placeholders(
1082
self,
1040
- mm_placeholders: Mapping[str, list[PlaceholderInfo]],
1083
+ mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
1084
mm_item_counts: Mapping[str, int],
1085
*,
1086
allow_missing: bool = False,