@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8acd80548
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[2/N] handling placeholders in merged multi-modal processor (#10485)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 23, 2024 at 13:25 UTC c8acd80548c77bd5d6302353708dd16ea705f031
5 files changed +975 -147
tests/multimodal/test_processing.py new
+370
@@ -0,0 +1,370 @@
1 +from typing import cast
2 +
3 +import pytest
4 +from transformers import BatchFeature
5 +
6 +from vllm.multimodal.processing import (PromptReplacement, find_text_matches,
7 + find_token_matches, iter_token_matches,
8 + iter_token_runs, replace_text_matches)
9 +from vllm.transformers_utils.tokenizer import AnyTokenizer
10 +from vllm.utils import full_groupby
11 +
12 +
13 +# yapf: disable
14 +@pytest.mark.parametrize(
15 + ("token_ids", "expected"),
16 + [
17 + ([], []),
18 + (
19 + [32000, 32000, 32000],
20 + [{ "token_id": 32000, "start_idx": 0, "length": 3 }],
21 + ),
22 + (
23 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
24 + [
25 + { "token_id": 9833, "start_idx": 0, "length": 1 },
26 + { "token_id": 28747, "start_idx": 1, "length": 1 },
27 + { "token_id": 32000, "start_idx": 2, "length": 3 },
28 + { "token_id": 9833, "start_idx": 5, "length": 1 },
29 + { "token_id": 28747, "start_idx": 6, "length": 1 },
30 + { "token_id": 32000, "start_idx": 7, "length": 2 },
31 + { "token_id": 918, "start_idx": 9, "length": 1 },
32 + ],
33 + ),
34 + ],
35 +)
36 +# yapf: enable
37 +def test_iter_token_runs(token_ids, expected):
38 + result = list(iter_token_runs(token_ids))
39 +
40 + # Only displayed on error
41 + print("result:", result)
42 +
43 + # Manually constructed results
44 + assert [item._asdict() for item in result] == expected
45 +
46 + # Invariants
47 + assert sum(run_info.length for run_info in result) == len(token_ids)
48 +
49 +
50 +# yapf: disable
51 +@pytest.mark.parametrize(
52 + ("token_ids", "match_ids", "expected"),
53 + [
54 + ([], [], [{ "start_idx": 0, "end_idx": 0 }]),
55 + ([], [32000], []),
56 + (
57 + [32000, 32000, 32000],
58 + [32000],
59 + [
60 + { "start_idx": 0, "end_idx": 1 },
61 + { "start_idx": 1, "end_idx": 2 },
62 + { "start_idx": 2, "end_idx": 3 },
63 + ],
64 + ),
65 + (
66 + [32000, 32000, 32000],
67 + [32000, 32000],
68 + [{ "start_idx": 0, "end_idx": 2 }],
69 + ),
70 + (
71 + [32000, 32000, 32000],
72 + [32000, 32000, 32000],
73 + [{ "start_idx": 0, "end_idx": 3 }],
74 + ),
75 + (
76 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
77 + [28747, 32000],
78 + [
79 + { "start_idx": 1, "end_idx": 3 },
80 + { "start_idx": 6, "end_idx": 8 },
81 + ],
82 + ),
83 + (
84 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
85 + [28747, 32000, 32000, 32000],
86 + [
87 + { "start_idx": 1, "end_idx": 5 },
88 + ],
89 + ),
90 + (
91 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
92 + [28747, 0, 32000],
93 + [],
94 + ),
95 + ],
96 +)
97 +# yapf: enable
98 +def test_iter_token_matches(token_ids, match_ids, expected):
99 + result = list(iter_token_matches(token_ids, match_ids))
100 +
101 + # Manually constructed results
102 + assert [item._asdict() for item in result] == expected
103 +
104 + # Invariants
105 + match_lens = [end - start for start, end in result]
106 + print("match_lens:", match_lens) # Only displayed on error
107 + assert all(match_len == len(match_ids) for match_len in match_lens)
108 +
109 +
110 +# yapf: disable
111 +@pytest.mark.parametrize(
112 + ("prompt", "target_by_key", "expected_by_key"),
113 + [
114 + (
115 + [],
116 + {
117 + "pattern_1": [],
118 + "pattern_2": [32000],
119 + },
120 + {
121 + "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
122 + "pattern_2": [],
123 + }
124 + ),
125 + (
126 + [32000, 32000, 32000, 32000],
127 + {
128 + "pattern_1": [32000],
129 + "pattern_2": [32000, 32000],
130 + "pattern_3": [32000, 32000, 32000],
131 + },
132 + {
133 + "pattern_1": [
134 + { "start_idx": 0, "end_idx": 1 },
135 + { "start_idx": 1, "end_idx": 2 },
136 + { "start_idx": 2, "end_idx": 3 },
137 + { "start_idx": 3, "end_idx": 4 },
138 + ],
139 + "pattern_2": [
140 + { "start_idx": 0, "end_idx": 2 },
141 + { "start_idx": 2, "end_idx": 4 },
142 + ],
143 + "pattern_3": [
144 + { "start_idx": 0, "end_idx": 3 },
145 + ],
146 + },
147 + ),
148 + (
149 + [9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918],
150 + {
151 + "pattern_1": [28747, 32000],
152 + "pattern_2": [28747, 32000, 32000, 32000],
153 + "pattern_3": [28747, 0, 32000],
154 + },
155 + {
156 + "pattern_1": [
157 + { "start_idx": 1, "end_idx": 3 },
158 + { "start_idx": 6, "end_idx": 8 },
159 + ],
160 + "pattern_2": [
161 + { "start_idx": 1, "end_idx": 5 },
162 + ],
163 + "pattern_3": [],
164 + },
165 + ),
166 + ],
167 +)
168 +# yapf: enable
169 +def test_find_token_matches(prompt, target_by_key, expected_by_key):
170 + # Should not be used since there is nothing to convert to token IDs
171 + mock_tokenizer = cast(AnyTokenizer, object())
172 +
173 + result = find_token_matches(
174 + prompt,
175 + [
176 + PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
177 + for key, target in target_by_key.items()
178 + ],
179 + )
180 +
181 + # Only displayed on error
182 + print("result:", result)
183 +
184 + # Manually constructed results
185 + result_groups = dict(full_groupby(result, key=lambda x: x.modality))
186 + assert {
187 + key: [
188 + dict(start_idx=item.start_idx, end_idx=item.end_idx)
189 + for item in result_groups.get(key, [])
190 + ]
191 + for key in expected_by_key
192 + } == expected_by_key
193 +
194 +
195 +# yapf: disable
196 +@pytest.mark.parametrize(
197 + ("prompt", "target_by_key", "expected_by_key"),
198 + [
199 + # Detokenized test cases of `test_find_token_matches`
200 + # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
201 + (
202 + "",
203 + {
204 + "pattern_1": "",
205 + "pattern_2": "<image>",
206 + },
207 + {
208 + "pattern_1": [{ "start_idx": 0, "end_idx": 0 }],
209 + "pattern_2": [],
210 + }
211 + ),
212 + (
213 + "<image><image><image><image>",
214 + {
215 + "pattern_1": "<image>",
216 + "pattern_2": "<image><image>",
217 + "pattern_3": "<image><image><image>",
218 + },
219 + {
220 + "pattern_1": [
221 + { "start_idx": 0, "end_idx": 7 },
222 + { "start_idx": 7, "end_idx": 14 },
223 + { "start_idx": 14, "end_idx": 21 },
224 + { "start_idx": 21, "end_idx": 28 },
225 + ],
226 + "pattern_2": [
227 + { "start_idx": 0, "end_idx": 14 },
228 + { "start_idx": 14, "end_idx": 28 },
229 + ],
230 + "pattern_3": [
231 + { "start_idx": 0, "end_idx": 21 },
232 + ],
233 + },
234 + ),
235 + (
236 + "Image:<image><image><image>Image:<image><image>!",
237 + {
238 + "pattern_1": "Image:<image>",
239 + "pattern_2": "Image:<image><image><image>",
240 + "pattern_3": "Image:<unk><image>",
241 + },
242 + {
243 + "pattern_1": [
244 + { "start_idx": 0, "end_idx": 13 },
245 + { "start_idx": 27, "end_idx": 40 },
246 + ],
247 + "pattern_2": [
248 + { "start_idx": 0, "end_idx": 27 },
249 + ],
250 + "pattern_3": [],
251 + },
252 + ),
253 + # Test regex escape
254 + (
255 + "<|image|><image><|image|><image>",
256 + {
257 + "pattern_1": "<|image|>",
258 + "pattern_2": "<|image|><image>",
259 + "pattern_3": "<|image|><image><|image|>",
260 + },
261 + {
262 + "pattern_1": [
263 + { "start_idx": 0, "end_idx": 9 },
264 + { "start_idx": 16, "end_idx": 25 },
265 + ],
266 + "pattern_2": [
267 + { "start_idx": 0, "end_idx": 16 },
268 + { "start_idx": 16, "end_idx": 32 },
269 + ],
270 + "pattern_3": [
271 + { "start_idx": 0, "end_idx": 25 },
272 + ],
273 + },
274 + ),
275 + ],
276 +)
277 +# yapf: enable
278 +def test_find_text_matches(prompt, target_by_key, expected_by_key):
279 + # Should not be used since there is nothing to convert to text
280 + mock_tokenizer = cast(AnyTokenizer, object())
281 +
282 + result = find_text_matches(
283 + prompt,
284 + [
285 + PromptReplacement(target, [], 0).bind(key, mock_tokenizer)
286 + for key, target in target_by_key.items()
287 + ],
288 + )
289 +
290 + # Only displayed on error
291 + print("result:", result)
292 +
293 + # Manually constructed results
294 + result_groups = dict(full_groupby(result, key=lambda x: x.modality))
295 + assert {
296 + key: [
297 + dict(start_idx=item.start_idx, end_idx=item.end_idx)
298 + for item in result_groups.get(key, [])
299 + ]
300 + for key in expected_by_key
301 + } == expected_by_key
302 +
303 +
304 +# yapf: disable
305 +@pytest.mark.parametrize(
306 + ("prompt", "target_by_key", "repl_by_key", "expected_by_mm_count"),
307 + [
308 + (
309 + "Image:<image>Image:<image><image>!",
310 + {
311 + # We use `<image>` before `Image:` to test matches that
312 + # occur out of order
313 + "pattern_1": "<image>",
314 + "pattern_2": "Image:",
315 + "pattern_3": "!",
316 + },
317 + {
318 + # Test whether target is confused with repl_unit
319 + "pattern_1": ("<image><image>", 1),
320 + # Test empty repl_unit
321 + "pattern_2": ("", 1),
322 + # Test multiple repl_count
323 + "pattern_3": ("?", 2),
324 + },
325 + {
326 + # Test no replacement
327 + 0: "Image:<image>Image:<image><image>!",
328 + # Test single replacement
329 + 1: "<image><image>Image:<image><image>??",
330 + # Test repeated replacement
331 + 2: "<image><image><image><image><image>??",
332 + },
333 + ),
334 + ]
335 +)
336 +# yapf: enable
337 +def test_find_replace_text(
338 + prompt,
339 + target_by_key,
340 + repl_by_key,
341 + expected_by_mm_count,
342 +):
343 + # Should not be used since there is nothing to convert to text
344 + mock_tokenizer = cast(AnyTokenizer, object())
345 +
346 + matches = find_text_matches(
347 + prompt,
348 + [
349 + PromptReplacement(target, *repl_by_key[key]) \
350 + .bind(key, mock_tokenizer)
351 + for key, target in target_by_key.items()
352 + ],
353 + )
354 + result_by_mm_count = {
355 + mm_count: replace_text_matches(
356 + prompt,
357 + matches,
358 + {key: list(range(mm_count))
359 + for key in repl_by_key},
360 + BatchFeature(),
361 + )
362 + for mm_count in expected_by_mm_count
363 + }
364 +
365 + # Only displayed on error
366 + print("matches:", matches)
367 + print("result_by_mm_count:", result_by_mm_count)
368 +
369 + # Manually constructed results
370 + assert result_by_mm_count == expected_by_mm_count
tests/multimodal/test_utils.py
+2 -1
@@ -139,7 +139,8 @@ def test_repeat_and_pad_placeholder_tokens(model):
139 2,
140 "<image><image><image>",
141 [32000, 32000, 32000],
142 - [{ "offset": 0, "length": 2 }]),
142 + [{ "offset": 0, "length": 2 }],
143 + ),
144 (
145 "<image><image>",
146 [3, 2],
vllm/multimodal/inputs.py
+1 -8
@@ -203,14 +203,7 @@ class MultiModalInputsV2(TypedDict):
203 """The type of inputs."""
204
205 prompt: str
206 - """
207 - The original, unprocessed prompt text.
208 -
209 - Note:
210 - Since prompt text is not required by vLLM internals, we leave this
211 - unprocessed to save CPU computation. You can still call
212 - :code:`tokenizer.decode(prompt_token_ids)` to get the processed text.
213 - """
206 + """The processed prompt text."""
207
208 prompt_token_ids: List[int]
209 """The processed token IDs which includes placeholder tokens."""
vllm/multimodal/processing.py
+583 -137
@@ -1,34 +1,91 @@
1 +import re
2 +from abc import ABC, abstractmethod
3 +from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4 from dataclasses import dataclass
2 -from functools import lru_cache, partial
3 -from typing import (Any, Callable, Collection, Generic, List, Mapping,
4 - Optional, TypedDict, TypeVar, final)
5 +from functools import lru_cache
6 +from itertools import groupby
7 +from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
8
9 +import numpy as np
10 from transformers import BatchFeature
7 -from typing_extensions import TypeAlias
11 +from typing_extensions import TypeAlias, TypedDict
12
13 from vllm.inputs import InputProcessingContext
14 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
11 -from vllm.utils import is_list_of
15 +from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
16
17 from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
18 MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
19 VideoItem)
20
21 +
22 +def bind_prompt_sequence(
23 + seq: Union[str, list[int]],
24 + tokenizer: AnyTokenizer,
25 +) -> "_BoundPromptSequence":
26 + """
27 + Bind a text or token sequence to a tokenizer so that it can be
28 + lazily converted into the other format on demand.
29 + """
30 + return _BoundPromptSequence(
31 + tokenizer=tokenizer,
32 + _text=seq if isinstance(seq, str) else None,
33 + _token_ids=seq if isinstance(seq, list) else None,
34 + )
35 +
36 +
37 _T = TypeVar("_T")
38 +_S = TypeVar("_S", str, list[int])
39
19 -ReplacementFunc: TypeAlias = Callable[[_T, BatchFeature, int], List[int]]
20 -"""
21 -Given the original data item, HF-processed data, and index of the processed
22 -item, output the replacement token IDs to be allocated in vLLM.
23 -"""
40 +
41 +@dataclass
42 +class PromptReplacement(Generic[_S, _T]):
43 + target: _S
44 + """The text or token sequence to find and replace."""
45 +
46 + repl_unit: _S
47 + """
48 + The unit making up the replacement text or token sequence.
49 +
50 + See :code:`repl_count` for more details.
51 + """
52 +
53 + repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
54 + """
55 + Given the original multi-modal items for this modality, HF-processed data,
56 + and index of the processed item, output the number of repetitions of
57 + :code:`repl_unit` to build up the replacement text or token sequence.
58 +
59 + For convenience, you can pass in an integer if the number of repetitions is
60 + a constant.
61 + """
62 +
63 + def __repr__(self) -> str:
64 + return (f"{type(self).__name__}(target={self.target!r}, "
65 + f"repl_unit={self.repl_unit!r})")
66 +
67 + def bind(
68 + self,
69 + modality: str,
70 + tokenizer: AnyTokenizer,
71 + ) -> "_BoundPromptReplacement[_T]":
72 + return _BoundPromptReplacement(
73 + modality=modality,
74 + target=bind_prompt_sequence(self.target, tokenizer),
75 + repl_unit=bind_prompt_sequence(self.repl_unit, tokenizer),
76 + repl_count=self.repl_count,
77 + )
78
79
80 @dataclass
81 class ModalityProcessingMetadata(Generic[_T]):
28 - placeholder_replacements: Mapping[str, ReplacementFunc]
82 + prompt_repls: Sequence[Union[PromptReplacement[str, _T],
83 + PromptReplacement[list[int], _T]]]
84 """
30 - A dictionary where each item represents the original placeholder in the
31 - prompt text and the corresponding replacement.
85 + Defines each text or token sequence to replace in the HF-processed prompt.
86 +
87 + This is skipped if the HF-processed prompt is found to already contain
88 + the replacement prompts.
89 """
90
91
@@ -52,46 +109,138 @@ Note:
109 Read more on that :ref:`here <adding_multimodal_plugin>`.
110 """
111
55 -MultiModalMultiData: TypeAlias = List[_T]
56 -"""
57 -A list of data items, where the number of data items allowed
58 -per modality is restricted by :code:`--limit-mm-per-prompt`.
59 -"""
112
113 +def _encode(
114 + tokenizer: AnyTokenizer,
115 + text: str,
116 + *,
117 + add_special_tokens: bool = False,
118 +) -> list[int]:
119 + """
120 + Backend-agnostic equivalent of HF's
121 + :code:`tokenizer.encode(text, add_special_tokens=...)`.
122 + """
123 + if isinstance(tokenizer, MistralTokenizer):
124 + return tokenizer.tokenizer.encode(text,
125 + bos=add_special_tokens,
126 + eos=add_special_tokens)
127
62 -@final
63 -class MultiModalMultiDataBuiltins(TypedDict, total=False):
64 - """Type annotations for modality types predefined by vLLM."""
128 + return tokenizer.encode(text, add_special_tokens=add_special_tokens)
129
66 - image: MultiModalMultiData[ImageItem]
67 - """The input images."""
130
69 - video: MultiModalMultiData[VideoItem]
70 - """The input videos."""
131 +@lru_cache(maxsize=2048)
132 +def _cached_encode(
133 + tokenizer: AnyTokenizer,
134 + text: str,
135 + *,
136 + add_special_tokens: bool = False,
137 +) -> list[int]:
138 + return _encode(tokenizer, text, add_special_tokens=add_special_tokens)
139
72 - audio: MultiModalMultiData[AudioItem]
73 - """The input audios."""
140
141 +def _decode(
142 + tokenizer: AnyTokenizer,
143 + token_ids: list[int],
144 + *,
145 + skip_special_tokens: bool = False,
146 +) -> str:
147 + """
148 + Backend-agnostic equivalent of HF's
149 + :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
150 + """
151 + return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
152
76 -MultiModalMultiDataDict: TypeAlias = Mapping[str, MultiModalMultiData[Any]]
77 -"""
78 -A dictionary containing an entry for each modality type to input.
153
80 -Note:
81 - This dictionary also accepts modality keys defined outside
82 - :class:`MultiModalMultiDataBuiltins` as long as a customized plugin
83 - is registered through the :class:`~vllm.multimodal.MULTIMODAL_REGISTRY`.
84 - Read more on that :ref:`here <adding_multimodal_plugin>`.
85 -"""
154 +@lru_cache(maxsize=2048)
155 +def _cached_decode(
156 + tokenizer: AnyTokenizer,
157 + token_ids: tuple[int, ...],
158 + *,
159 + skip_special_tokens: bool = False,
160 +) -> str:
161 + return _decode(tokenizer,
162 + list(token_ids),
163 + skip_special_tokens=skip_special_tokens)
164 +
165 +
166 +class _HasModalityAttr(Protocol):
167 + modality: str
168 +
169
170 +class _HasModalityProp(Protocol):
171
88 -def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
172 + @property
173 + def modality(self) -> str:
174 + ...
175 +
176 +
177 +_M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
178 +
179 +
180 +def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
181 + """Convenience function to apply :func:`full_groupby` based on modality."""
182 + return full_groupby(values, key=lambda x: x.modality)
183 +
184 +
185 +@dataclass
186 +class _BoundPromptSequence:
187 + tokenizer: AnyTokenizer
188 + _text: Optional[str]
189 + _token_ids: Optional[list[int]]
190 +
191 + def __post_init__(self) -> None:
192 + if self._text is None and self._token_ids is None:
193 + raise ValueError("At least one of 'text' and 'token_ids' must be "
194 + "specified")
195 +
196 + @property
197 + def text(self) -> str:
198 + if self._text is None:
199 + assert self._token_ids is not None
200 + self._text = _cached_decode(self.tokenizer, tuple(self._token_ids))
201 +
202 + return self._text
203 +
204 + @property
205 + def token_ids(self) -> list[int]:
206 + if self._token_ids is None:
207 + assert self._text is not None
208 + self._token_ids = _cached_encode(self.tokenizer, self._text)
209 +
210 + return self._token_ids
211 +
212 + def __repr__(self) -> str:
213 + return (f"{type(self).__name__}(_text={self._text!r}, "
214 + f"_token_ids={self._token_ids!r})")
215 +
216 +
217 +@dataclass
218 +class _BoundPromptReplacement(Generic[_T]):
219 + modality: str
220 + target: _BoundPromptSequence
221 + repl_unit: _BoundPromptSequence
222 + repl_count: Union[Callable[[list[_T], BatchFeature, int], int], int]
223 +
224 + def get_count(
225 + self,
226 + mm_items: list[_T],
227 + hf_inputs: BatchFeature,
228 + item_idx: int,
229 + ) -> int:
230 + repl_count = self.repl_count
231 + if isinstance(repl_count, int):
232 + return repl_count
233 +
234 + return repl_count(mm_items, hf_inputs, item_idx)
235 +
236 +
237 +def to_multi_format(data: MultiModalDataDict) -> dict[str, list[Any]]:
238 """
239 Convert a :class:`MultiModalDataDict` containing single data items
240 to a :class:`MultiModalMultiDataDict` containing multiple data items
241 per entry.
242 """
94 - multi_data: Mapping[str, MultiModalMultiData[Any]] = {}
243 + multi_data = dict[str, list[Any]]()
244
245 for k, v in data.items():
246 # yapf: disable
@@ -107,86 +256,279 @@ def to_multi_format(data: MultiModalDataDict) -> MultiModalMultiDataDict:
256 return multi_data
257
258
110 -def encode_no_special_tokens(
111 - tokenizer: AnyTokenizer,
112 - text: str,
113 -) -> List[int]:
259 +class _TokenRun(NamedTuple):
260 + token_id: int
261 +
262 + start_idx: int
263 + length: int
264 +
265 +
266 +def iter_token_runs(token_ids: list[int]) -> Iterable[_TokenRun]:
267 """
115 - Backend-agnostic equivalent of HF's
116 - :code:`tokenizer.encode(text, add_special_tokens=False)`.
268 + Yield the starting index and length of each run of tokens that are the same.
269 """
118 - if isinstance(tokenizer, MistralTokenizer):
119 - return tokenizer.tokenizer.encode(text, bos=False, eos=False)
270 + start_idx = 0
271 +
272 + for token_id, it in groupby(token_ids):
273 + length = sum(1 for _ in it)
274 + yield _TokenRun(token_id=token_id, start_idx=start_idx, length=length)
275 +
276 + start_idx += length
277 +
278 +
279 +class _PlaceholderInfo(NamedTuple):
280 + modality: str
281 + offset: int
282 + length: int
283 +
284 + def to_range(self) -> PlaceholderRange:
285 + return PlaceholderRange(offset=self.offset, length=self.length)
286 +
287 +
288 +def iter_placeholders(
289 + prompt_repls: Sequence[_BoundPromptReplacement[Any]],
290 + token_ids: list[int],
291 + *,
292 + min_placeholder_count: int,
293 +) -> Iterable[_PlaceholderInfo]:
294 + """Yield each set of placeholder tokens found in :code:`token_ids`."""
295 + placeholder_ids_by_modality = {
296 + modality: {
297 + token_id
298 + for prompt_repl in repls
299 + for token_id in prompt_repl.repl_unit.token_ids
300 + }
301 + for modality, repls in full_groupby_modality(prompt_repls)
302 + }
303
121 - return tokenizer.encode(text, add_special_tokens=False)
304 + for run_info in iter_token_runs(token_ids):
305 + if run_info.length > min_placeholder_count:
306 + for (modality,
307 + placeholder_ids) in placeholder_ids_by_modality.items():
308 + if run_info.token_id in placeholder_ids:
309 + yield _PlaceholderInfo(
310 + modality=modality,
311 + offset=run_info.start_idx,
312 + length=run_info.length,
313 + )
314
315
124 -@lru_cache
125 -def candidate_placeholders(
126 - tokenizer: AnyTokenizer,
127 - placeholder_text: str,
128 -) -> Collection[List[int]]:
129 - """Generate token ID sequences that may represent a placeholder text."""
130 - # When the placeholder text is not mapped to a special token ID,
131 - # it may be tokenized differently based on whether it is at the start/end
132 - # of the string. So, we go through each combination of whether the text
133 - # is at the start and end boundaries of the string
134 -
135 - # Matches the placeholder when it is in the middle of the string
136 - start_id, = encode_no_special_tokens(tokenizer, "a")
137 - end_id, = encode_no_special_tokens(tokenizer, "b")
138 -
139 - candidate_basic = encode_no_special_tokens(tokenizer, placeholder_text)
140 -
141 - start_id_, *candidate_a = encode_no_special_tokens(
142 - tokenizer,
143 - f"a{placeholder_text}",
144 - )
145 - assert start_id == start_id_
316 +class _TokenMatch(NamedTuple):
317 + start_idx: int
318 + end_idx: int
319
147 - start_id_, *candidate_ab, end_id_ = encode_no_special_tokens(
148 - tokenizer,
149 - f"a{placeholder_text}b",
150 - )
151 - assert start_id == start_id_ and end_id == end_id_
320
153 - *candidate_b, end_id_ = encode_no_special_tokens(
154 - tokenizer,
155 - f"{placeholder_text}b",
156 - )
157 - assert end_id == end_id_
321 +def iter_token_matches(
322 + token_ids: list[int],
323 + match_ids: list[int],
324 +) -> Iterable[_TokenMatch]:
325 + """Yield each occurrence of :code:`match_ids` in :code:`token_ids`."""
326 + match_len = len(match_ids)
327
159 - # Remove duplicates (need to convert to tuple to be hashable)
160 - unique_candidates = {
161 - tuple(c)
162 - for c in [candidate_basic, candidate_a, candidate_ab, candidate_b]
163 - }
328 + last_end_idx = 0
329 + for start_idx in range(len(token_ids) - match_len + 1):
330 + if start_idx < last_end_idx:
331 + continue # Exclude overlapping matches
332
165 - # Convert back to list
166 - return [list(c) for c in unique_candidates]
333 + end_idx = start_idx + match_len
334 + if token_ids[start_idx:end_idx] == match_ids:
335 + yield _TokenMatch(start_idx=start_idx, end_idx=end_idx)
336 + last_end_idx = end_idx
337
338
169 -def apply_placeholders(
170 - token_ids: List[int],
171 - placeholder_ids: List[int],
172 - get_replacement_ids: Callable[[], List[int]],
173 -) -> Optional[PlaceholderRange]:
174 - """
175 - Find the first occurrence of :code:`placeholder_ids`,
176 - and replace it with the output of :code:`get_replacement_ids`.
339 +class _PromptReplacementMatch(ABC, Generic[_T, _S]):
340 + prompt_repl: _BoundPromptReplacement[_T]
341 +
342 + @property
343 + def modality(self) -> str:
344 + return self.prompt_repl.modality
345 +
346 + @property
347 + @abstractmethod
348 + def start_idx(self) -> int:
349 + raise NotImplementedError
350 +
351 + @property
352 + @abstractmethod
353 + def end_idx(self) -> int:
354 + raise NotImplementedError
355 +
356 + @abstractmethod
357 + def get_repl(
358 + self,
359 + mm_items: list[_T],
360 + hf_inputs: BatchFeature,
361 + item_idx: int,
362 + ) -> _S:
363 + raise NotImplementedError
364 +
365 + def __repr__(self) -> str:
366 + return (f"{type(self).__name__}(modality={self.modality!r}, "
367 + f"start_idx={self.start_idx!r}, end_idx={self.end_idx!r})")
368 +
369 +
370 +@dataclass(repr=False)
371 +class _PromptReplacementTokenMatch(_PromptReplacementMatch[_T, list[int]]):
372 + prompt_repl: _BoundPromptReplacement[_T]
373 + match: _TokenMatch
374 +
375 + @property
376 + def start_idx(self) -> int:
377 + return self.match.start_idx
378 +
379 + @property
380 + def end_idx(self) -> int:
381 + return self.match.end_idx
382 +
383 + def get_repl(
384 + self,
385 + mm_items: list[_T],
386 + hf_inputs: BatchFeature,
387 + item_idx: int,
388 + ) -> list[int]:
389 + prompt_repl = self.prompt_repl
390 + count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
391 + return prompt_repl.repl_unit.token_ids * count
392
178 - This function updates :code:`token_ids` in place.
393 +
394 +@dataclass(repr=False)
395 +class _PromptReplacementTextMatch(_PromptReplacementMatch[_T, str]):
396 + prompt_repl: _BoundPromptReplacement[_T]
397 + match: re.Match[str]
398 +
399 + @property
400 + def start_idx(self) -> int:
401 + return self.match.start()
402 +
403 + @property
404 + def end_idx(self) -> int:
405 + return self.match.end()
406 +
407 + def get_repl(
408 + self,
409 + mm_items: list[_T],
410 + hf_inputs: BatchFeature,
411 + item_idx: int,
412 + ) -> str:
413 + prompt_repl = self.prompt_repl
414 + count = prompt_repl.get_count(mm_items, hf_inputs, item_idx)
415 + return prompt_repl.repl_unit.text * count
416 +
417 +
418 +def find_token_matches(
419 + prompt: list[int],
420 + prompt_repls: Sequence[_BoundPromptReplacement[_T]],
421 +) -> list[_PromptReplacementTokenMatch[_T]]:
422 + """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
423 + return [
424 + _PromptReplacementTokenMatch(prompt_repl, match)
425 + for prompt_repl in prompt_repls
426 + for match in iter_token_matches(prompt, prompt_repl.target.token_ids)
427 + ]
428 +
429 +
430 +def find_text_matches(
431 + prompt: str,
432 + prompt_repls: Sequence[_BoundPromptReplacement[_T]],
433 +) -> list[_PromptReplacementTextMatch[_T]]:
434 + """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
435 + return [
436 + _PromptReplacementTextMatch(prompt_repl, match)
437 + for prompt_repl in prompt_repls
438 + for match in re.finditer(re.escape(prompt_repl.target.text), prompt)
439 + ]
440 +
441 +
442 +def _resolve_matches(
443 + prompt: _S,
444 + matches: Sequence[_PromptReplacementMatch[_T, _S]],
445 +) -> list[_PromptReplacementMatch[_T, _S]]:
446 + """
447 + Resolve :code:`matches` to ensure that there are no overlapping matches,
448 + and sort them such that earlier matches take priority over later ones.
449 """
180 - placeholder_length = len(placeholder_ids)
450 + num_matches_by_idx = np.zeros(len(prompt), dtype=int)
451 + for match in matches:
452 + num_matches_by_idx[match.start_idx:match.end_idx] += 1
453 +
454 + duplicate_matches_idxs, = np.nonzero(num_matches_by_idx > 1)
455 + if len(duplicate_matches_idxs) > 0:
456 + raise ValueError("Unable to find a unique replacement "
457 + f"at indices={duplicate_matches_idxs} "
458 + f"of prompt={prompt}")
459 +
460 + return sorted(matches, key=lambda x: x.start_idx)
461 +
462 +
463 +def _replace_matches(
464 + prompt: _S,
465 + matches: Sequence[_PromptReplacementMatch[_T, _S]],
466 + mm_items_by_modality: Mapping[str, list[_T]],
467 + hf_inputs: BatchFeature,
468 +) -> list[_S]:
469 + out_seqs = list[_S]()
470 + prev_end_idx = 0
471 + next_idx_by_modality = {modality: 0 for modality in mm_items_by_modality}
472 +
473 + for match in _resolve_matches(prompt, matches):
474 + modality = match.modality
475 + mm_items = mm_items_by_modality[modality]
476 +
477 + item_idx = next_idx_by_modality[modality]
478 + if item_idx >= len(mm_items):
479 + continue
480 +
481 + start_idx = match.start_idx
482 + end_idx = match.end_idx
483 + repl_ids = match.get_repl(mm_items, hf_inputs, item_idx)
484 +
485 + out_seqs.append(prompt[prev_end_idx:start_idx] + repl_ids)
486 + prev_end_idx = end_idx
487 + next_idx_by_modality[modality] += 1
488 +
489 + out_seqs.append(prompt[prev_end_idx:])
490 +
491 + return out_seqs
492 +
493 +
494 +def replace_token_matches(
495 + prompt: list[int],
496 + matches: Sequence[_PromptReplacementMatch[_T, list[int]]],
497 + mm_items_by_modality: Mapping[str, list[_T]],
498 + hf_inputs: BatchFeature,
499 +) -> list[int]:
500 + """Apply :code:`prompt_repls` to :code:`prompt`."""
501 + if not matches:
502 + return prompt
503 +
504 + token_id_seqs = _replace_matches(
505 + prompt,
506 + matches,
507 + mm_items_by_modality,
508 + hf_inputs,
509 + )
510 +
511 + return flatten_2d_lists(token_id_seqs)
512
182 - for start_idx in range(len(token_ids) - placeholder_length + 1):
183 - if token_ids[start_idx:placeholder_length] == placeholder_ids:
184 - token_ids[start_idx:placeholder_length] = get_replacement_ids()
513
186 - return PlaceholderRange(offset=start_idx,
187 - length=placeholder_length)
514 +def replace_text_matches(
515 + prompt: str,
516 + matches: Sequence[_PromptReplacementMatch[_T, str]],
517 + mm_items_by_modality: Mapping[str, list[_T]],
518 + hf_inputs: BatchFeature,
519 +) -> str:
520 + """Apply :code:`prompt_repls` to :code:`prompt`."""
521 + if not matches:
522 + return prompt
523
189 - return None
524 + texts = _replace_matches(
525 + prompt,
526 + matches,
527 + mm_items_by_modality,
528 + hf_inputs,
529 + )
530 +
531 + return "".join(texts)
532
533
534 class MultiModalProcessor:
@@ -212,62 +554,166 @@ class MultiModalProcessor:
554 ) -> MultiModalInputsV2:
555 return self.apply(prompt, mm_data, mm_processor_kwargs)
556
215 - def apply(
557 + def _find_placeholders(
558 + self,
559 + all_prompt_repls: Sequence[_BoundPromptReplacement[Any]],
560 + new_token_ids: list[int],
561 + *,
562 + # To avoid false positives from multi-input when detecting
563 + # whether placeholder tokens have been inserted, in case
564 + # the target sequence is a subset of the replacement tokens
565 + min_placeholder_count: int = 16,
566 + ) -> list[_PlaceholderInfo]:
567 + return list(
568 + iter_placeholders(
569 + all_prompt_repls,
570 + new_token_ids,
571 + min_placeholder_count=min_placeholder_count,
572 + ))
573 +
574 + def _apply_hf_processor(
575 self,
576 prompt: str,
577 mm_data: MultiModalDataDict,
578 mm_processor_kwargs: Mapping[str, object],
220 - ) -> MultiModalInputsV2:
221 - tokenizer = self.ctx.tokenizer
579 + ) -> BatchFeature:
580 hf_processor = self.ctx.get_hf_processor()
581
224 - processed_inputs = hf_processor(
582 + return hf_processor(
583 text=prompt, # type: ignore
584 **mm_data,
585 **mm_processor_kwargs,
586 )
229 - new_token_ids, = processed_inputs.pop("input_ids").tolist()
230 - mm_kwargs = MultiModalKwargs(processed_inputs)
587
232 - mm_placeholders: Mapping[str, List[PlaceholderRange]] = {}
588 + def _bind_prompt_replacements(
589 + self,
590 + mm_data: MultiModalDataDict,
591 + ) -> list[_BoundPromptReplacement[Any]]:
592 + tokenizer = self.ctx.tokenizer
593
234 - for modality, orig_inputs in to_multi_format(mm_data).items():
235 - assert isinstance(orig_inputs, list)
594 + return [
595 + prompt_repl.bind(modality, tokenizer)
596 + for modality, metadata in self.metadata.items()
597 + if modality in mm_data for prompt_repl in metadata.prompt_repls
598 + ]
599
237 - metadata = self.metadata[modality]
238 - placeholder_replacements = metadata.placeholder_replacements
600 + def _apply_prompt_replacements(
601 + self,
602 + mm_data: MultiModalDataDict,
603 + hf_inputs: BatchFeature,
604 + token_ids: list[int],
605 + prompt_repls: Sequence[_BoundPromptReplacement[Any]],
606 + ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
607 + tokenizer = self.ctx.tokenizer
608
240 - modality_placeholders: List[PlaceholderRange] = []
609 + mm_items = to_multi_format(mm_data)
610 + token_matches = find_token_matches(token_ids, prompt_repls)
611 +
612 + # If the search text does not represent a special token,
613 + # it may have different token IDs in the prompt, because
614 + # the tokens may go across the boundaries of the search text.
615 + # ----
616 + # e.g. when searching for "foo" in "food", if "food" itself makes
617 + # up a token, then the token ID of "foo" will not appear at all
618 + # ----
619 + # Since it is inefficient to search for all possible tokenizations
620 + # of the search text in the prompt, we instead perform string
621 + # replacement on the decoded token IDs, then encode them back.
622 + if all(
623 + len(matches) >= len(mm_data[modality])
624 + for modality, matches in full_groupby_modality(token_matches)
625 + ): # yapf: disable
626 + token_ids = replace_token_matches(
627 + token_ids,
628 + token_matches,
629 + mm_items,
630 + hf_inputs,
631 + )
632 +
633 + text = _decode(tokenizer, token_ids)
634 + matched_repls = [match.prompt_repl for match in token_matches]
635 + else:
636 + text = _decode(tokenizer, token_ids)
637 +
638 + text_matches = find_text_matches(text, prompt_repls)
639 + text = replace_text_matches(
640 + text,
641 + text_matches,
642 + mm_items,
643 + hf_inputs,
644 + )
645 +
646 + token_ids = _encode(tokenizer, text)
647 + matched_repls = [match.prompt_repl for match in text_matches]
648 +
649 + placeholders = self._find_placeholders(matched_repls, token_ids)
650 +
651 + # Sanity check
652 + assert len(placeholders) == len(matched_repls), dict(
653 + # Log this information for easier debugging
654 + text=text,
655 + token_ids=token_ids,
656 + placeholders=placeholders,
657 + matched_repls=matched_repls,
658 + )
659
242 - for item_idx, orig_item in enumerate(orig_inputs):
243 - for match_text, replace_fn in placeholder_replacements.items():
244 - candidates = candidate_placeholders(tokenizer, match_text)
245 - get_replacement_ids = partial(
246 - replace_fn,
247 - orig_item,
248 - processed_inputs,
249 - item_idx,
250 - )
660 + return token_ids, text, placeholders
661
252 - for match_ids in candidates:
253 - # TODO(youkaichao): Don't update new_token_ids
254 - placeholders = apply_placeholders(
255 - new_token_ids,
256 - match_ids,
257 - get_replacement_ids,
258 - )
662 + def apply(
663 + self,
664 + prompt_text: str,
665 + mm_data: MultiModalDataDict,
666 + mm_processor_kwargs: Mapping[str, object],
667 + ) -> MultiModalInputsV2:
668 + """
669 + Process multi-modal inputs to be used in vLLM.
670 +
671 + The main steps are:
672 +
673 + 1. Apply HF Processor on prompt text and multi-modal data together,
674 + outputting token IDs and processed tensors.
675 + 2. Find and replace sequences in the token IDs with placeholder tokens.
676 + The number of placeholder tokens equals the feature size of the
677 + multi-modal data outputted by the multi-modal encoder.
678 + 3. Extract information about the placeholder tokens from the
679 + processed token IDs.
680 + """
681 + tokenizer = self.ctx.tokenizer
682 +
683 + hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
684 + mm_processor_kwargs)
685 + prompt_ids, = hf_inputs.pop("input_ids").tolist()
686 + mm_kwargs = MultiModalKwargs(hf_inputs)
687
260 - if placeholders is not None:
261 - modality_placeholders.append(placeholders)
688 + all_prompt_repls = self._bind_prompt_replacements(mm_data)
689
263 - # yapf: disable
264 - mm_placeholders[modality] = modality_placeholders # type: ignore[index]
265 - # yapf: enable
690 + # If HF processor already inserts placeholder tokens,
691 + # there is no need for us to insert them
692 + all_placeholders = self._find_placeholders(all_prompt_repls,
693 + prompt_ids)
694 + if all_placeholders:
695 + prompt_text = _decode(tokenizer, prompt_ids)
696 + else:
697 + (
698 + prompt_ids,
699 + prompt_text,
700 + all_placeholders,
701 + ) = self._apply_prompt_replacements(
702 + mm_data,
703 + hf_inputs,
704 + prompt_ids,
705 + all_prompt_repls,
706 + )
707 +
708 + mm_placeholders = {
709 + modality: [item.to_range() for item in items]
710 + for modality, items in full_groupby_modality(all_placeholders)
711 + }
712
713 return MultiModalInputsV2(
714 type="multimodal",
269 - prompt=prompt,
270 - prompt_token_ids=new_token_ids,
715 + prompt=prompt_text,
716 + prompt_token_ids=prompt_ids,
717 mm_kwargs=mm_kwargs,
718 mm_placeholders=mm_placeholders,
719 )
vllm/utils.py
+19 -1
@@ -19,7 +19,8 @@ import uuid
19 import warnings
20 import weakref
21 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
22 -from collections.abc import Mapping
22 +from collections import defaultdict
23 +from collections.abc import Iterable, Mapping
24 from functools import lru_cache, partial, wraps
25 from platform import uname
26 from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
@@ -905,6 +906,23 @@ def flatten_2d_lists(lists: List[List[T]]) -> List[T]:
906 return [item for sublist in lists for item in sublist]
907
908
909 +_K = TypeVar("_K", bound=Hashable)
910 +_V = TypeVar("_V")
911 +
912 +
913 +def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
914 + """
915 + Unlike :class:`itertools.groupby`, groups are not broken by
916 + non-contiguous data.
917 + """
918 + groups = defaultdict[_K, list[_V]](list)
919 +
920 + for value in values:
921 + groups[key(value)].append(value)
922 +
923 + return groups.items()
924 +
925 +
926 # TODO: This function can be removed if transformer_modules classes are
927 # serialized by value when communicating between processes
928 def init_cached_hf_modules() -> None: