A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[2/N] handling placeholders in merged multi-modal processor (#10485)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 23, 2024 at 13:25 UTC
c8acd80548c77bd5d6302353708dd16ea705f031
5 files changed
+975
-147
tests/multimodal/test_processing.py
new
+370
@@ -0,0 +1,370 @@
1
+from typing import cast
2
+
3
+import pytest
4
+from transformers import BatchFeature
5
+
6
+from vllm.multimodal.processing import (PromptReplacement, find_text_matches,
7
+ find_token_matches, iter_token_matches,
8
+ iter_token_runs, replace_text_matches)
9
+from vllm.transformers_utils.tokenizer import AnyTokenizer
10
+from vllm.utils import full_groupby
11
+
12
+
13
+# yapf: disable
14
+@pytest.mark.parametrize(
15
+ ("token_ids", "expected"),
16
+ [
17
+ ([], []),
18
+ (
19
+ [32000, 32000, 32000],
20
+ [{ "token_id": 32000, "start_idx": 0, "length": 3 }],
21
+ ),
22
+ (
23
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
24
+ [
25
+ { "token_id": 9833, "start_idx": 0, "length": 1 },
26
+ { "token_id": 28747, "start_idx": 1, "length": 1 },
27
+ { "token_id": 32000, "start_idx": 2, "length": 3 },
28
+ { "token_id": 9833, "start_idx": 5, "length": 1 },
29
+ { "token_id": 28747, "start_idx": 6, "length": 1 },
30
+ { "token_id": 32000, "start_idx": 7, "length": 2 },
31
+ { "token_id": 918, "start_idx": 9, "length": 1 },
32
+ ],
33
+ ),
34
+ ],
35
+)
36
+# yapf: enable
37
+def test_iter_token_runs(token_ids, expected):
38
+ result = list(iter_token_runs(token_ids))
39
+
40
+ # Only displayed on error
41
+ print("result:", result)
42
+
43
+ # Manually constructed results
44
+ assert [item._asdict() for item in result] == expected
45
+
46
+ # Invariants
47
+ assert sum(run_info.length for run_info in result) == len(token_ids)
48
+
49
+
50
+# yapf: disable
51
+@pytest.mark.parametrize(
52
+ ("token_ids", "match_ids", "expected"),
53
+ [
54
+ ([], [], [{ "start_idx": 0, "end_idx": 0 }]),
55
+ ([], [32000], []),
56
+ (
57
+ [32000, 32000, 32000],
58
+ [32000],
59
+ [
60
+ { "start_idx": 0, "end_idx": 1 },
61
+ { "start_idx": 1, "end_idx": 2 },
62
+ { "start_idx": 2, "end_idx": 3 },
63
+ ],
64
+ ),
65
+ (
66
+ [32000, 32000, 32000],
67
+ [32000, 32000],
68
+ [{ "start_idx": 0, "end_idx": 2 }],
69
+ ),
70
+ (
71
+ [32000, 32000, 32000],
72
+ [32000, 32000, 32000],
73
+ [{ "start_idx": 0, "end_idx": 3 }],
74
+ ),
75
+ (
76
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
77
+ [28747, 32000],
78
+ [
79
+ { "start_idx": 1, "end_idx": 3 },
80
+ { "start_idx": 6, "end_idx": 8 },
81
+ ],
82
+ ),
83
+ (
84
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
85
+ [28747, 32000, 32000, 32000],
86
+ [
87
+ { "start_idx": 1, "end_idx": 5 },
88
+ ],
89
+ ),
90
+ (
91
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
92
+ [28747, 0, 32000],
93
+ [],
94
+ ),
95
+ ],
96
+)
97
+# yapf: enable
98
+def test_iter_token_matches(token_ids, match_ids, expected):
99
+ result = list(iter_token_matches(token_ids, match_ids))
100
+
101
+ # Manually constructed results
102
+ assert [item._asdict() for item in result] == expected
103
+
104
+ # Invariants
105
+ match_lens = [end - start for start, end in result]
106
+ print("match_lens:", match_lens) # Only displayed on error
107
+ assert all(match_len == len(match_ids) for match_len in match_lens)
108
+
109
+
110
+# yapf: disable
111
+@pytest.mark.parametrize(
112
+ ("prompt", "target_by_key", "expected_by_key"),
113
+ [
114
+ (
115
+ [],
116
+ {
117
+ "pattern_1": [],
118
+ "pattern_2": [32000],
119
+ },
120
+ {
121
+ "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
122
+ "pattern_2": [],
123
+ }
124
+ ),
125
+ (
126
+ [32000, 32000, 32000, 32000],
127
+ {
128
+ "pattern_1": [32000],
129
+ "pattern_2": [32000, 32000],
130
+ "pattern_3": [32000, 32000, 32000],
131
+ },
132
+ {
133
+ "pattern_1": [
134
+ { "start_idx": 0, "end_idx": 1 },
135
+ { "start_idx": 1, "end_idx": 2 },
136
+ { "start_idx": 2, "end_idx": 3 },
137
+ { "start_idx": 3, "end_idx": 4 },
138
+ ],
139
+ "pattern_2": [
140
+ { "start_idx": 0, "end_idx": 2 },
141
+ { "start_idx": 2, "end_idx": 4 },
142
+ ],
143
+ "pattern_3": [
144
+ { "start_idx": 0, "end_idx": 3 },
145
+ ],
146
+ },
147
+ ),
148
+ (
149
+ [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
150
+ {
151
+ "pattern_1": [28747, 32000],
152
+ "pattern_2": [28747, 32000, 32000, 32000],
153
+ "pattern_3": [28747, 0, 32000],
154
+ },
155
+ {
156
+ "pattern_1": [
157
+ { "start_idx": 1, "end_idx": 3 },
158
+ { "start_idx": 6, "end_idx": 8 },
159
+ ],
160
+ "pattern_2": [
161
+ { "start_idx": 1, "end_idx": 5 },
162
+ ],
163
+ "pattern_3": [],
164
+ },
165
+ ),
166
+ ],
167
+)
168
+# yapf: enable
169
+def test_find_token_matches(prompt, target_by_key, expected_by_key):
170
+ # Should not be used since there is nothing to convert to token IDs
171
+ mock_tokenizer = cast(AnyTokenizer, object())
172
+
173
+ result = find_token_matches(
174
+ prompt,
175
+ [
176
+ PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
177
+ for key, target in target_by_key.items()
178
+ ],
179
+ )
180
+
181
+ # Only displayed on error
182
+ print("result:", result)
183
+
184
+ # Manually constructed results
185
+ result_groups = dict(full_groupby(result, key=lambda x: x.modality))
186
+ assert {
187
+ key: [
188
+ dict(start_idx=item.start_idx, end_idx=item.end_idx)
189
+ for item in result_groups.get(key, [])
190
+ ]
191
+ for key in expected_by_key
192
+ } == expected_by_key
193
+
194
+
195
+# yapf: disable
196
+@pytest.mark.parametrize(
197
+ ("prompt", "target_by_key", "expected_by_key"),
198
+ [
199
+ # Detokenized test cases of `test_find_token_matches`
200
+ # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
201
+ (
202
+ "",
203
+ {
204
+ "pattern_1": "",
205
+ "pattern_2": "<image>",
206
+ },
207
+ {
208
+ "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
209
+ "pattern_2": [],
210
+ }
211
+ ),
212
+ (
213
+ "<image><image><image><image>",
214
+ {
215
+ "pattern_1": "<image>",
216
+ "pattern_2": "<image><image>",
217
+ "pattern_3": "<image><image><image>",
218
+ },
219
+ {
220
+ "pattern_1": [
221
+ { "start_idx": 0, "end_idx": 7 },
222
+ { "start_idx": 7, "end_idx": 14 },
223
+ { "start_idx": 14, "end_idx": 21 },
224
+ { "start_idx": 21, "end_idx": 28 },
225
+ ],
226
+ "pattern_2": [
227
+ { "start_idx": 0, "end_idx": 14 },
228
+ { "start_idx": 14, "end_idx": 28 },
229
+ ],
230
+ "pattern_3": [
231
+ { "start_idx": 0, "end_idx": 21 },
232
+ ],
233
+ },
234
+ ),
235
+ (
236
+ "Image:<image><image><image>Image:<image><image>!",
237
+ {
238
+ "pattern_1": "Image:<image>",
239
+ "pattern_2": "Image:<image><image><image>",
240
+ "pattern_3": "Image:<unk><image>",
241
+ },
242
+ {
243
+ "pattern_1": [
244
+ { "start_idx": 0, "end_idx": 13 },
245
+ { "start_idx": 27, "end_idx": 40 },
246
+ ],
247
+ "pattern_2": [
248
+ { "start_idx": 0, "end_idx": 27 },
249
+ ],
250
+ "pattern_3": [],
251
+ },
252
+ ),
253
+ # Test regex escape
254
+ (
255
+ "<|image|><image><|image|><image>",
256
+ {
257
+ "pattern_1": "<|image|>",
258
+ "pattern_2": "<|image|><image>",
259
+ "pattern_3": "<|image|><image><|image|>",
260
+ },
261
+ {
262
+ "pattern_1": [
263
+ { "start_idx": 0, "end_idx": 9 },
264
+ { "start_idx": 16, "end_idx": 25 },
265
+ ],
266
+ "pattern_2": [
267
+ { "start_idx": 0, "end_idx": 16 },
268
+ { "start_idx": 16, "end_idx": 32 },
269
+ ],
270
+ "pattern_3": [
271
+ { "start_idx": 0, "end_idx": 25 },
272
+ ],
273
+ },
274
+ ),
275
+ ],
276
+)
277
+# yapf: enable
278
+def test_find_text_matches(prompt, target_by_key, expected_by_key):
279
+ # Should not be used since there is nothing to convert to text
280
+ mock_tokenizer = cast(AnyTokenizer, object())
281
+
282
+ result = find_text_matches(
283
+ prompt,
284
+ [
285
+ PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
286
+ for key, target in target_by_key.items()
287
+ ],
288
+ )
289
+
290
+ # Only displayed on error
291
+ print("result:", result)
292
+
293
+ # Manually constructed results
294
+ result_groups = dict(full_groupby(result, key=lambda x: x.modality))
295
+ assert {
296
+ key: [
297
+ dict(start_idx=item.start_idx, end_idx=item.end_idx)
298
+ for item in result_groups.get(key, [])
299
+ ]
300
+ for key in expected_by_key
301
+ } == expected_by_key
302
+
303
+
304
+# yapf: disable
305
+@pytest.mark.parametrize(
306
+ ("prompt", "target_by_key", "repl_by_key", "expected_by_mm_count"),
307
+ [
308
+ (
309
+ "Image:<image>Image:<image><image>!",
310
+ {
311
+ # We use `<image>` before `Image:` to test matches that
312
+ # occur out of order
313
+ "pattern_1": "<image>",
314
+ "pattern_2": "Image:",
315
+ "pattern_3": "!",
316
+ },
317
+ {
318
+ # Test whether target is confused with repl_unit
319
+ "pattern_1": ("<image><image>", 1),
320
+ # Test empty repl_unit
321
+ "pattern_2": ("", 1),
322
+ # Test multiple repl_count
323
+ "pattern_3": ("?", 2),
324
+ },
325
+ {
326
+ # Test no replacement
327
+ 0: "Image:<image>Image:<image><image>!",
328
+ # Test single replacement
329
+ 1: "<image><image>Image:<image><image>??",
330
+ # Test repeated replacement
331
+ 2: "<image><image><image><image><image>??",
332
+ },
333
+ ),
334
+ ]
335
+)
336
+# yapf: enable
337
+def test_find_replace_text(
338
+ prompt,
339
+ target_by_key,
340
+ repl_by_key,
341
+ expected_by_mm_count,
342
+):
343
+ # Should not be used since there is nothing to convert to text
344
+ mock_tokenizer = cast(AnyTokenizer, object())
345
+
346
+ matches = find_text_matches(
347
+ prompt,
348
+ [
349
+ PromptReplacement(target, *repl_by_key[key]) \
350
+ .bind(key, mock_tokenizer)
351
+ for key, target in target_by_key.items()
352
+ ],
353
+ )
354
+ result_by_mm_count = {
355
+ mm_count: replace_text_matches(
356
+ prompt,
357
+ matches,
358
+ {key: list(range(mm_count))
359
+ for key in repl_by_key},
360
+ BatchFeature(),
361
+ )
362
+ for mm_count in expected_by_mm_count
363
+ }
364
+
365
+ # Only displayed on error
366
+ print("matches:", matches)
367
+ print("result_by_mm_count:", result_by_mm_count)
368
+
369
+ # Manually constructed results
370
+ assert result_by_mm_count == expected_by_mm_count
tests/multimodal/test_utils.py
+2
-1
@@ -139,7 +139,8 @@ def test_repeat_and_pad_placeholder_tokens(model):
139
2,
140
"<image><image><image>",
141
[32000, 32000, 32000],
142
- [{ "offset": 0, "length": 2 }]),
142
+ [{ "offset": 0, "length": 2 }],
143
+ ),
144
(
145
"<image><image>",
146
[3, 2],
vllm/multimodal/inputs.py
+1
-8
@@ -203,14 +203,7 @@ class MultiModalInputsV2(TypedDict):
203
"""The type of inputs."""
204
205
prompt: str
206
- """
207
- The original, unprocessed prompt text.
208
-
209
- Note:
210
- Since prompt text is not required by vLLM internals, we leave this
211
- unprocessed to save CPU computation. You can still call
212
- :code:`tokenizer.decode(prompt_token_ids)` to get the processed text.
213
- """
206
+ """The processed prompt text."""
207
208
prompt_token_ids: List[int]
209
"""The processed token IDs which includes placeholder tokens."""
vllm/multimodal/processing.py
+583
-137
@@ -1,34 +1,91 @@
1
+import re
2
+from abc import ABC, abstractmethod
3
+from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4
from dataclasses import dataclass
2
-from functools import lru_cache, partial
3
-from typing import (Any, Callable, Collection, Generic, List, Mapping,
4
- Optional, TypedDict, TypeVar, final)
5
+from functools import lru_cache
6
+from itertools import groupby
7
+from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
8
9
+import numpy as np
10
from transformers import BatchFeature
7
-from typing_extensions import TypeAlias
11
+from typing_extensions import TypeAlias, TypedDict
12
13
from vllm.inputs import InputProcessingContext
14
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
11
-from vllm.utils import is_list_of
15
+from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
16
17
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
18
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
19
VideoItem)
20
21
+
22
+def bind_prompt_sequence(
23
+ seq: Union[str, list[int]],
24
+ tokenizer: AnyTokenizer,
25
+) -> "_BoundPromptSequence":
26
+ """
27
+ Bind a text or token sequence to a tokenizer so that it can be
28
+ lazily converted into the other format on demand.
29
+ """
30
+ return _BoundPromptSequence(
31
+ tokenizer=tokenizer,
32
+ _text=seq if isinstance(seq, str) else None,
33
+ _token_ids=seq if isinstance(seq, list) else None,
34
+ )
35
+
36
+
37
_T = TypeVar("_T")
38
+_S = TypeVar("_S", str, list[int])
39
19
-ReplacementFunc: TypeAlias = Callable[[_T, BatchFeature, int], List[int]]
20
-"""
21
-Given the original data item, HF-processed data, and index of the processed
22
-item, output the replacement token IDs to be allocated in vLLM.
23
-"""
40
+
41
+@dataclass
42
+class PromptReplacement(Generic[_S, _T]):
43
+ target: _S
44
+ """The text or token sequence to find and replace."""
45
+
46
+ repl_unit: _S
47
+ """
48
+ The unit making up the replacement text or token sequence.
49
+
50
+ See :code:`repl_count` for more details.
51
+ """
52
+
53
+ repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
54
+ """
55
+ Given the original multi-modal items for this modality, HF-processed data,
56
+ and index of the processed item, output the number of repetitions of
57
+ :code:`repl_unit` to build up the replacement text or token sequence.
58
+
59
+ For convenience, you can pass in an integer if the number of repetitions is
60
+ a constant.
61
+ """
62
+
63
+ def __repr__(self) -> str:
64
+ return (f"{type(self).__name__}(target={self.target!r}, "
65
+ f"repl_unit={self.repl_unit!r})")
66
+
67
+ def bind(
68
+ self,
69
+ modality: str,
70
+ tokenizer: AnyTokenizer,
71
+ ) -> "_BoundPromptReplacement[_T]":
72
+ return _BoundPromptReplacement(
73
+ modality=modality,
74
+ target=bind_prompt_sequence(self.target, tokenizer),
75
+ repl_unit=bind_prompt_sequence(self.repl_unit, tokenizer),
76
+ repl_count=self.repl_count,
77
+ )
78
79
80
@dataclass
81
class ModalityProcessingMetadata(Generic[_T]):
28
- placeholder_replacements: Mapping[str, ReplacementFunc]
82
+ prompt_repls: Sequence[Union[PromptReplacement[str, _T],
83
+ PromptReplacement[list[int], _T]]]
84
"""
30
- A dictionary where each item represents the original placeholder in the
31
- prompt text and the corresponding replacement.
85
+ Defines each text or token sequence to replace in the HF-processed prompt.
86
+
87
+ This is skipped if the HF-processed prompt is found to already contain
88
+ the replacement prompts.
89
"""
90
91
@@ -52,46 +109,138 @@ Note:
109
Read more on that :ref:`here <adding_multimodal_plugin>`.
110
"""
111
55
-MultiModalMultiData: TypeAlias = List[_T]
56
-"""
57
-A list of data items, where the number of data items allowed
58
-per modality is restricted by :code:`--limit-mm-per-prompt`.
59
-"""
112
113
+def _encode(
114
+ tokenizer: AnyTokenizer,
115
+ text: str,
116
+ *,
117
+ add_special_tokens: bool = False,
118
+) -> list[int]:
119
+ """
120
+ Backend-agnostic equivalent of HF's
121
+ :code:`tokenizer.encode(text, add_special_tokens=...)`.
122
+ """
123
+ if isinstance(tokenizer, MistralTokenizer):
124
+ return tokenizer.tokenizer.encode(text,
125
+ bos=add_special_tokens,
126
+ eos=add_special_tokens)
127
62
-@final
63
-class MultiModalMultiDataBuiltins(TypedDict, total=False):
64
- """Type annotations for modality types predefined by vLLM."""
128
+ return tokenizer.encode(text, add_special_tokens=add_special_tokens)
129
66
- image: MultiModalMultiData[ImageItem]
67
- """The input images."""
130
69
- video: MultiModalMultiData[VideoItem]
70
- """The input videos."""
131
+@lru_cache(maxsize=2048)
132
+def _cached_encode(
133
+ tokenizer: AnyTokenizer,
134
+ text: str,
135
+ *,
136
+ add_special_tokens: bool = False,
137
+) -> list[int]:
138
+ return _encode(tokenizer, text, add_special_tokens=add_special_tokens)
139
72
- audio: MultiModalMultiData[AudioItem]
73
- """The input audios."""
140
141
+def _decode(
142
+ tokenizer: AnyTokenizer,
143
+ token_ids: list[int],
144
+ *,
145
+ skip_special_tokens: bool = False,
146
+) -> str:
147
+ """
148
+ Backend-agnostic equivalent of HF's
149
+ :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
150
+ """
151
+ return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
152
76
-MultiModalMultiDataDict: TypeAlias = Mapping[str, MultiModalMultiData[Any]]
77
-"""
78
-A dictionary containing an entry for each modality type to input.
153
80
-Note:
81
- This dictionary also accepts modality keys defined outside
82
- :class:`MultiModalMultiDataBuiltins` as long as a customized plugin
83
- is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
84
- Read more on that :ref:`here <adding_multimodal_plugin>`.
85
-"""
154
+@lru_cache(maxsize=2048)
155
+def _cached_decode(
156
+ tokenizer: AnyTokenizer,
157
+ token_ids: tuple[int, ...],
158
+ *,
159
+ skip_special_tokens: bool = False,
160
+) -> str:
161
+ return _decode(tokenizer,
162
+ list(token_ids),
163
+ skip_special_tokens=skip_special_tokens)
164
+
165
+
166
+class _HasModalityAttr(Protocol):
167
+ modality: str
168
+
169
170
+class _HasModalityProp(Protocol):
171
88
-def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
172
+ @property
173
+ def modality(self) -> str:
174
+ ...
175
+
176
+
177
+_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
178
+
179
+
180
+def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
181
+ """Convenience function to apply :func:`full_groupby` based on modality."""
182
+ return full_groupby(values, key=lambda x: x.modality)
183
+
184
+
185
+@dataclass
186
+class _BoundPromptSequence:
187
+ tokenizer: AnyTokenizer
188
+ _text: Optional[str]
189
+ _token_ids: Optional[list[int]]
190
+
191
+ def __post_init__(self) -> None:
192
+ if self._text is None and self._token_ids is None:
193
+ raise ValueError("At least one of 'text' and 'token_ids' must be "
194
+ "specified")
195
+
196
+ @property
197
+ def text(self) -> str:
198
+ if self._text is None:
199
+ assert self._token_ids is not None
200
+ self._text = _cached_decode(self.tokenizer, tuple(self._token_ids))
201
+
202
+ return self._text
203
+
204
+ @property
205
+ def token_ids(self) -> list[int]:
206
+ if self._token_ids is None:
207
+ assert self._text is not None
208
+ self._token_ids = _cached_encode(self.tokenizer, self._text)
209
+
210
+ return self._token_ids
211
+
212
+ def __repr__(self) -> str:
213
+ return (f"{type(self).__name__}(_text={self._text!r}, "
214
+ f"_token_ids={self._token_ids!r})")
215
+
216
+
217
+@dataclass
218
+class _BoundPromptReplacement(Generic[_T]):
219
+ modality: str
220
+ target: _BoundPromptSequence
221
+ repl_unit: _BoundPromptSequence
222
+ repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
223
+
224
+ def get_count(
225
+ self,
226
+ mm_items: list[_T],
227
+ hf_inputs: BatchFeature,
228
+ item_idx: int,
229
+ ) -> int:
230
+ repl_count = self.repl_count
231
+ if isinstance(repl_count, int):
232
+ return repl_count
233
+
234
+ return repl_count(mm_items, hf_inputs, item_idx)
235
+
236
+
237
+def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
238
"""
239
Convert a :class:`MultiModalDataDict` containing single data items
240
to a :class:`MultiModalMultiDataDict` containing multiple data items
241
per entry.
242
"""
94
- multi_data: Mapping[str, MultiModalMultiData[Any]] = {}
243
+ multi_data = dict[str, list[Any]]()
244
245
for k, v in data.items():
246
# yapf: disable
@@ -107,86 +256,279 @@ def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
256
return multi_data
257
258
110
-def encode_no_special_tokens(
111
- tokenizer: AnyTokenizer,
112
- text: str,
113
-) -> List[int]:
259
+class _TokenRun(NamedTuple):
260
+ token_id: int
261
+
262
+ start_idx: int
263
+ length: int
264
+
265
+
266
+def iter_token_runs(token_ids: list[int]) -> Iterable[_TokenRun]:
267
"""
115
- Backend-agnostic equivalent of HF's
116
- :code:`tokenizer.encode(text, add_special_tokens=False)`.
268
+ Yield the starting index and length of each run of tokens that are the same.
269
"""
118
- if isinstance(tokenizer, MistralTokenizer):
119
- return tokenizer.tokenizer.encode(text, bos=False, eos=False)
270
+ start_idx = 0
271
+
272
+ for token_id, it in groupby(token_ids):
273
+ length = sum(1 for _ in it)
274
+ yield _TokenRun(token_id=token_id, start_idx=start_idx, length=length)
275
+
276
+ start_idx += length
277
+
278
+
279
+class _PlaceholderInfo(NamedTuple):
280
+ modality: str
281
+ offset: int
282
+ length: int
283
+
284
+ def to_range(self) -> PlaceholderRange:
285
+ return PlaceholderRange(offset=self.offset, length=self.length)
286
+
287
+
288
+def iter_placeholders(
289
+ prompt_repls: Sequence[_BoundPromptReplacement[Any]],
290
+ token_ids: list[int],
291
+ *,
292
+ min_placeholder_count: int,
293
+) -> Iterable[_PlaceholderInfo]:
294
+ """Yield each set of placeholder tokens found in :code:`token_ids`."""
295
+ placeholder_ids_by_modality = {
296
+ modality: {
297
+ token_id
298
+ for prompt_repl in repls
299
+ for token_id in prompt_repl.repl_unit.token_ids
300
+ }
301
+ for modality, repls in full_groupby_modality(prompt_repls)
302
+ }
303
121
- return tokenizer.encode(text, add_special_tokens=False)
304
+ for run_info in iter_token_runs(token_ids):
305
+ if run_info.length > min_placeholder_count:
306
+ for (modality,
307
+ placeholder_ids) in placeholder_ids_by_modality.items():
308
+ if run_info.token_id in placeholder_ids:
309
+ yield _PlaceholderInfo(
310
+ modality=modality,
311
+ offset=run_info.start_idx,
312
+ length=run_info.length,
313
+ )
314
315
124
-@lru_cache
125
-def candidate_placeholders(
126
- tokenizer: AnyTokenizer,
127
- placeholder_text: str,
128
-) -> Collection[List[int]]:
129
- """Generate token ID sequences that may represent a placeholder text."""
130
- # When the placeholder text is not mapped to a special token ID,
131
- # it may be tokenized differently based on whether it is at the start/end
132
- # of the string. So, we go through each combination of whether the text
133
- # is at the start and end boundaries of the string
134
-
135
- # Matches the placeholder when it is in the middle of the string
136
- start_id, = encode_no_special_tokens(tokenizer, "a")
137
- end_id, = encode_no_special_tokens(tokenizer, "b")
138
-
139
- candidate_basic = encode_no_special_tokens(tokenizer, placeholder_text)
140
-
141
- start_id_, *candidate_a = encode_no_special_tokens(
142
- tokenizer,
143
- f"a{placeholder_text}",
144
- )
145
- assert start_id == start_id_
316
+class _TokenMatch(NamedTuple):
317
+ start_idx: int
318
+ end_idx: int
319
147
- start_id_, *candidate_ab, end_id_ = encode_no_special_tokens(
148
- tokenizer,
149
- f"a{placeholder_text}b",
150
- )
151
- assert start_id == start_id_ and end_id == end_id_
320
153
- *candidate_b, end_id_ = encode_no_special_tokens(
154
- tokenizer,
155
- f"{placeholder_text}b",
156
- )
157
- assert end_id == end_id_
321
+def iter_token_matches(
322
+ token_ids: list[int],
323
+ match_ids: list[int],
324
+) -> Iterable[_TokenMatch]:
325
+ """Yield each occurrence of :code:`match_ids` in :code:`token_ids`."""
326
+ match_len = len(match_ids)
327
159
- # Remove duplicates (need to convert to tuple to be hashable)
160
- unique_candidates = {
161
- tuple(c)
162
- for c in [candidate_basic, candidate_a, candidate_ab, candidate_b]
163
- }
328
+ last_end_idx = 0
329
+ for start_idx in range(len(token_ids) - match_len + 1):
330
+ if start_idx < last_end_idx:
331
+ continue # Exclude overlapping matches
332
165
- # Convert back to list
166
- return [list(c) for c in unique_candidates]
333
+ end_idx = start_idx + match_len
334
+ if token_ids[start_idx:end_idx] == match_ids:
335
+ yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
336
+ last_end_idx = end_idx
337
338
169
-def apply_placeholders(
170
- token_ids: List[int],
171
- placeholder_ids: List[int],
172
- get_replacement_ids: Callable[[], List[int]],
173
-) -> Optional[PlaceholderRange]:
174
- """
175
- Find the first occurrence of :code:`placeholder_ids`,
176
- and replace it with the output of :code:`get_replacement_ids`.
339
+class _PromptReplacementMatch(ABC, Generic[_T, _S]):
340
+ prompt_repl: _BoundPromptReplacement[_T]
341
+
342
+ @property
343
+ def modality(self) -> str:
344
+ return self.prompt_repl.modality
345
+
346
+ @property
347
+ @abstractmethod
348
+ def start_idx(self) -> int:
349
+ raise NotImplementedError
350
+
351
+ @property
352
+ @abstractmethod
353
+ def end_idx(self) -> int:
354
+ raise NotImplementedError
355
+
356
+ @abstractmethod
357
+ def get_repl(
358
+ self,
359
+ mm_items: list[_T],
360
+ hf_inputs: BatchFeature,
361
+ item_idx: int,
362
+ ) -> _S:
363
+ raise NotImplementedError
364
+
365
+ def __repr__(self) -> str:
366
+ return (f"{type(self).__name__}(modality={self.modality!r}, "
367
+ f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
368
+
369
+
370
+@dataclass(repr=False)
371
+class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
372
+ prompt_repl: _BoundPromptReplacement[_T]
373
+ match: _TokenMatch
374
+
375
+ @property
376
+ def start_idx(self) -> int:
377
+ return self.match.start_idx
378
+
379
+ @property
380
+ def end_idx(self) -> int:
381
+ return self.match.end_idx
382
+
383
+ def get_repl(
384
+ self,
385
+ mm_items: list[_T],
386
+ hf_inputs: BatchFeature,
387
+ item_idx: int,
388
+ ) -> list[int]:
389
+ prompt_repl = self.prompt_repl
390
+ count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
391
+ return prompt_repl.repl_unit.token_ids * count
392
178
- This function updates :code:`token_ids` in place.
393
+
394
+@dataclass(repr=False)
395
+class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
396
+ prompt_repl: _BoundPromptReplacement[_T]
397
+ match: re.Match[str]
398
+
399
+ @property
400
+ def start_idx(self) -> int:
401
+ return self.match.start()
402
+
403
+ @property
404
+ def end_idx(self) -> int:
405
+ return self.match.end()
406
+
407
+ def get_repl(
408
+ self,
409
+ mm_items: list[_T],
410
+ hf_inputs: BatchFeature,
411
+ item_idx: int,
412
+ ) -> str:
413
+ prompt_repl = self.prompt_repl
414
+ count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
415
+ return prompt_repl.repl_unit.text * count
416
+
417
+
418
+def find_token_matches(
419
+ prompt: list[int],
420
+ prompt_repls: Sequence[_BoundPromptReplacement[_T]],
421
+) -> list[_PromptReplacementTokenMatch[_T]]:
422
+ """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
423
+ return [
424
+ _PromptReplacementTokenMatch(prompt_repl, match)
425
+ for prompt_repl in prompt_repls
426
+ for match in iter_token_matches(prompt, prompt_repl.target.token_ids)
427
+ ]
428
+
429
+
430
+def find_text_matches(
431
+ prompt: str,
432
+ prompt_repls: Sequence[_BoundPromptReplacement[_T]],
433
+) -> list[_PromptReplacementTextMatch[_T]]:
434
+ """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
435
+ return [
436
+ _PromptReplacementTextMatch(prompt_repl, match)
437
+ for prompt_repl in prompt_repls
438
+ for match in re.finditer(re.escape(prompt_repl.target.text), prompt)
439
+ ]
440
+
441
+
442
+def _resolve_matches(
443
+ prompt: _S,
444
+ matches: Sequence[_PromptReplacementMatch[_T, _S]],
445
+) -> list[_PromptReplacementMatch[_T, _S]]:
446
+ """
447
+ Resolve :code:`matches` to ensure that there are no overlapping matches,
448
+ and sort them such that earlier matches take priority over later ones.
449
"""
180
- placeholder_length = len(placeholder_ids)
450
+ num_matches_by_idx = np.zeros(len(prompt), dtype=int)
451
+ for match in matches:
452
+ num_matches_by_idx[match.start_idx:match.end_idx] += 1
453
+
454
+ duplicate_matches_idxs, = np.nonzero(num_matches_by_idx > 1)
455
+ if len(duplicate_matches_idxs) > 0:
456
+ raise ValueError("Unable to find a unique replacement "
457
+ f"at indices={duplicate_matches_idxs} "
458
+ f"of prompt={prompt}")
459
+
460
+ return sorted(matches, key=lambda x: x.start_idx)
461
+
462
+
463
+def _replace_matches(
464
+ prompt: _S,
465
+ matches: Sequence[_PromptReplacementMatch[_T, _S]],
466
+ mm_items_by_modality: Mapping[str, list[_T]],
467
+ hf_inputs: BatchFeature,
468
+) -> list[_S]:
469
+ out_seqs = list[_S]()
470
+ prev_end_idx = 0
471
+ next_idx_by_modality = {modality: 0 for modality in mm_items_by_modality}
472
+
473
+ for match in _resolve_matches(prompt, matches):
474
+ modality = match.modality
475
+ mm_items = mm_items_by_modality[modality]
476
+
477
+ item_idx = next_idx_by_modality[modality]
478
+ if item_idx >= len(mm_items):
479
+ continue
480
+
481
+ start_idx = match.start_idx
482
+ end_idx = match.end_idx
483
+ repl_ids = match.get_repl(mm_items, hf_inputs, item_idx)
484
+
485
+ out_seqs.append(prompt[prev_end_idx:start_idx] + repl_ids)
486
+ prev_end_idx = end_idx
487
+ next_idx_by_modality[modality] += 1
488
+
489
+ out_seqs.append(prompt[prev_end_idx:])
490
+
491
+ return out_seqs
492
+
493
+
494
+def replace_token_matches(
495
+ prompt: list[int],
496
+ matches: Sequence[_PromptReplacementMatch[_T, list[int]]],
497
+ mm_items_by_modality: Mapping[str, list[_T]],
498
+ hf_inputs: BatchFeature,
499
+) -> list[int]:
500
+ """Apply :code:`prompt_repls` to :code:`prompt`."""
501
+ if not matches:
502
+ return prompt
503
+
504
+ token_id_seqs = _replace_matches(
505
+ prompt,
506
+ matches,
507
+ mm_items_by_modality,
508
+ hf_inputs,
509
+ )
510
+
511
+ return flatten_2d_lists(token_id_seqs)
512
182
- for start_idx in range(len(token_ids) - placeholder_length + 1):
183
- if token_ids[start_idx:placeholder_length] == placeholder_ids:
184
- token_ids[start_idx:placeholder_length] = get_replacement_ids()
513
186
- return PlaceholderRange(offset=start_idx,
187
- length=placeholder_length)
514
+def replace_text_matches(
515
+ prompt: str,
516
+ matches: Sequence[_PromptReplacementMatch[_T, str]],
517
+ mm_items_by_modality: Mapping[str, list[_T]],
518
+ hf_inputs: BatchFeature,
519
+) -> str:
520
+ """Apply :code:`prompt_repls` to :code:`prompt`."""
521
+ if not matches:
522
+ return prompt
523
189
- return None
524
+ texts = _replace_matches(
525
+ prompt,
526
+ matches,
527
+ mm_items_by_modality,
528
+ hf_inputs,
529
+ )
530
+
531
+ return "".join(texts)
532
533
534
class MultiModalProcessor:
@@ -212,62 +554,166 @@ class MultiModalProcessor:
554
) -> MultiModalInputsV2:
555
return self.apply(prompt, mm_data, mm_processor_kwargs)
556
215
- def apply(
557
+ def _find_placeholders(
558
+ self,
559
+ all_prompt_repls: Sequence[_BoundPromptReplacement[Any]],
560
+ new_token_ids: list[int],
561
+ *,
562
+ # To avoid false positives from multi-input when detecting
563
+ # whether placeholder tokens have been inserted, in case
564
+ # the target sequence is a subset of the replacement tokens
565
+ min_placeholder_count: int = 16,
566
+ ) -> list[_PlaceholderInfo]:
567
+ return list(
568
+ iter_placeholders(
569
+ all_prompt_repls,
570
+ new_token_ids,
571
+ min_placeholder_count=min_placeholder_count,
572
+ ))
573
+
574
+ def _apply_hf_processor(
575
self,
576
prompt: str,
577
mm_data: MultiModalDataDict,
578
mm_processor_kwargs: Mapping[str, object],
220
- ) -> MultiModalInputsV2:
221
- tokenizer = self.ctx.tokenizer
579
+ ) -> BatchFeature:
580
hf_processor = self.ctx.get_hf_processor()
581
224
- processed_inputs = hf_processor(
582
+ return hf_processor(
583
text=prompt, # type: ignore
584
**mm_data,
585
**mm_processor_kwargs,
586
)
229
- new_token_ids, = processed_inputs.pop("input_ids").tolist()
230
- mm_kwargs = MultiModalKwargs(processed_inputs)
587
232
- mm_placeholders: Mapping[str, List[PlaceholderRange]] = {}
588
+ def _bind_prompt_replacements(
589
+ self,
590
+ mm_data: MultiModalDataDict,
591
+ ) -> list[_BoundPromptReplacement[Any]]:
592
+ tokenizer = self.ctx.tokenizer
593
234
- for modality, orig_inputs in to_multi_format(mm_data).items():
235
- assert isinstance(orig_inputs, list)
594
+ return [
595
+ prompt_repl.bind(modality, tokenizer)
596
+ for modality, metadata in self.metadata.items()
597
+ if modality in mm_data for prompt_repl in metadata.prompt_repls
598
+ ]
599
237
- metadata = self.metadata[modality]
238
- placeholder_replacements = metadata.placeholder_replacements
600
+ def _apply_prompt_replacements(
601
+ self,
602
+ mm_data: MultiModalDataDict,
603
+ hf_inputs: BatchFeature,
604
+ token_ids: list[int],
605
+ prompt_repls: Sequence[_BoundPromptReplacement[Any]],
606
+ ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
607
+ tokenizer = self.ctx.tokenizer
608
240
- modality_placeholders: List[PlaceholderRange] = []
609
+ mm_items = to_multi_format(mm_data)
610
+ token_matches = find_token_matches(token_ids, prompt_repls)
611
+
612
+ # If the search text does not represent a special token,
613
+ # it may have different token IDs in the prompt, because
614
+ # the tokens may go across the boundaries of the search text.
615
+ # ----
616
+ # e.g. when searching for "foo" in "food", if "food" itself makes
617
+ # up a token, then the token ID of "foo" will not appear at all
618
+ # ----
619
+ # Since it is inefficient to search for all possible tokenizations
620
+ # of the search text in the prompt, we instead perform string
621
+ # replacement on the decoded token IDs, then encode them back.
622
+ if all(
623
+ len(matches) >= len(mm_data[modality])
624
+ for modality, matches in full_groupby_modality(token_matches)
625
+ ): # yapf: disable
626
+ token_ids = replace_token_matches(
627
+ token_ids,
628
+ token_matches,
629
+ mm_items,
630
+ hf_inputs,
631
+ )
632
+
633
+ text = _decode(tokenizer, token_ids)
634
+ matched_repls = [match.prompt_repl for match in token_matches]
635
+ else:
636
+ text = _decode(tokenizer, token_ids)
637
+
638
+ text_matches = find_text_matches(text, prompt_repls)
639
+ text = replace_text_matches(
640
+ text,
641
+ text_matches,
642
+ mm_items,
643
+ hf_inputs,
644
+ )
645
+
646
+ token_ids = _encode(tokenizer, text)
647
+ matched_repls = [match.prompt_repl for match in text_matches]
648
+
649
+ placeholders = self._find_placeholders(matched_repls, token_ids)
650
+
651
+ # Sanity check
652
+ assert len(placeholders) == len(matched_repls), dict(
653
+ # Log this information for easier debugging
654
+ text=text,
655
+ token_ids=token_ids,
656
+ placeholders=placeholders,
657
+ matched_repls=matched_repls,
658
+ )
659
242
- for item_idx, orig_item in enumerate(orig_inputs):
243
- for match_text, replace_fn in placeholder_replacements.items():
244
- candidates = candidate_placeholders(tokenizer, match_text)
245
- get_replacement_ids = partial(
246
- replace_fn,
247
- orig_item,
248
- processed_inputs,
249
- item_idx,
250
- )
660
+ return token_ids, text, placeholders
661
252
- for match_ids in candidates:
253
- # TODO(youkaichao): Don't update new_token_ids
254
- placeholders = apply_placeholders(
255
- new_token_ids,
256
- match_ids,
257
- get_replacement_ids,
258
- )
662
+ def apply(
663
+ self,
664
+ prompt_text: str,
665
+ mm_data: MultiModalDataDict,
666
+ mm_processor_kwargs: Mapping[str, object],
667
+ ) -> MultiModalInputsV2:
668
+ """
669
+ Process multi-modal inputs to be used in vLLM.
670
+
671
+ The main steps are:
672
+
673
+ 1. Apply HF Processor on prompt text and multi-modal data together,
674
+ outputting token IDs and processed tensors.
675
+ 2. Find and replace sequences in the token IDs with placeholder tokens.
676
+ The number of placeholder tokens equals the feature size of the
677
+ multi-modal data outputted by the multi-modal encoder.
678
+ 3. Extract information about the placeholder tokens from the
679
+ processed token IDs.
680
+ """
681
+ tokenizer = self.ctx.tokenizer
682
+
683
+ hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
684
+ mm_processor_kwargs)
685
+ prompt_ids, = hf_inputs.pop("input_ids").tolist()
686
+ mm_kwargs = MultiModalKwargs(hf_inputs)
687
260
- if placeholders is not None:
261
- modality_placeholders.append(placeholders)
688
+ all_prompt_repls = self._bind_prompt_replacements(mm_data)
689
263
- # yapf: disable
264
- mm_placeholders[modality] = modality_placeholders # type: ignore[index]
265
- # yapf: enable
690
+ # If HF processor already inserts placeholder tokens,
691
+ # there is no need for us to insert them
692
+ all_placeholders = self._find_placeholders(all_prompt_repls,
693
+ prompt_ids)
694
+ if all_placeholders:
695
+ prompt_text = _decode(tokenizer, prompt_ids)
696
+ else:
697
+ (
698
+ prompt_ids,
699
+ prompt_text,
700
+ all_placeholders,
701
+ ) = self._apply_prompt_replacements(
702
+ mm_data,
703
+ hf_inputs,
704
+ prompt_ids,
705
+ all_prompt_repls,
706
+ )
707
+
708
+ mm_placeholders = {
709
+ modality: [item.to_range() for item in items]
710
+ for modality, items in full_groupby_modality(all_placeholders)
711
+ }
712
713
return MultiModalInputsV2(
714
type="multimodal",
269
- prompt=prompt,
270
- prompt_token_ids=new_token_ids,
715
+ prompt=prompt_text,
716
+ prompt_token_ids=prompt_ids,
717
mm_kwargs=mm_kwargs,
718
mm_placeholders=mm_placeholders,
719
)
vllm/utils.py
+19
-1
@@ -19,7 +19,8 @@ import uuid
19
import warnings
20
import weakref
21
from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
22
-from collections.abc import Mapping
22
+from collections import defaultdict
23
+from collections.abc import Iterable, Mapping
24
from functools import lru_cache, partial, wraps
25
from platform import uname
26
from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -905,6 +906,23 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
906
return [item for sublist in lists for item in sublist]
907
908
909
+_K = TypeVar("_K", bound=Hashable)
910
+_V = TypeVar("_V")
911
+
912
+
913
+def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
914
+ """
915
+ Unlike :class:`itertools.groupby`, groups are not broken by
916
+ non-contiguous data.
917
+ """
918
+ groups = defaultdict[_K, list[_V]](list)
919
+
920
+ for value in values:
921
+ groups[key(value)].append(value)
922
+
923
+ return groups.items()
924
+
925
+
926
# TODO: This function can be removed if transformer_modules classes are
927
# serialized by value when communicating between processes
928
def init_cached_hf_modules() -> None: