@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6609cdf019
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Doc] Add docs for prompt replacement (#12318)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 22, 2025 at 22:56 UTC 6609cdf0195480f3d3bf1352a96acd6fd6ab89d8
2 files changed +79 -15
vllm/model_executor/models/ultravox.py
+1 -1
@@ -218,7 +218,7 @@ class UltravoxMultiModalProcessor(
218 return [
219 PromptReplacement(
220 modality="audio",
221 - target='<|audio|>',
221 + target="<|audio|>",
222 replacement=get_replacement_ultravox,
223 )
224 ]
vllm/multimodal/processing.py
+78 -14
@@ -29,41 +29,101 @@ if TYPE_CHECKING:
29 logger = init_logger(__name__)
30
31 _S = TypeVar("_S", str, list[int])
32 -_PromptSeq = Union[str, list[int]]
32 +
33 +PromptSeq = Union[str, list[int]]
34 +"""A token sequence (list of token IDs) or text."""
35
36
37 @dataclass
38 class PromptReplacementDetails:
37 - full: _PromptSeq
39 + """Details about the replacement token sequence or text."""
40 +
41 + full: PromptSeq
42 """The full replacement."""
43
40 - features: _PromptSeq
44 + features: PromptSeq
45 """
42 - The part of the replacement that corresponds to placeholder feature tokens.
46 + The part of the replacement that corresponds to feature placeholders;
47 + this will be replaced by the output of the vision encoder during model
48 + inference.
49 """
50
51 @staticmethod
46 - def from_seq(seq: _PromptSeq) -> "PromptReplacementDetails":
52 + def from_seq(seq: PromptSeq) -> "PromptReplacementDetails":
53 return PromptReplacementDetails(full=seq, features=seq)
54
55
50 -_PromptRepl = Union[_PromptSeq, PromptReplacementDetails]
56 +PromptRepl = Union[PromptSeq, PromptReplacementDetails]
57 +"""
58 +The replacement token sequence or text.
59 +
60 +If only part of the replacement corresponds to feature placeholders, you can
61 +use :class:`PromptReplacementDetails` to specify which part.
62 +"""
63
64
65 @dataclass
66 class PromptReplacement:
67 """
68 Defines how to replace portions of an input prompt with placeholder tokens.
69 +
70 + Example:
71 +
72 + For each image, replace one ``<image>`` input placeholder in the prompt
73 + with a number of ``<image>`` feature placeholders
74 + equal to the feature size of the vision encoder:
75 +
76 + .. code-block:: python
77 +
78 + PromptReplacement(
79 + modality="image",
80 + target="<image>",
81 + replacement="<image>" * image_feature_size,
82 + )
83 +
84 + As above, but further pad the feature placeholders with ``<image_bos>``
85 + and `<image_eos>``, which are not supposed to be passed to the vision
86 + encoder:
87 +
88 + .. code-block:: python
89 +
90 + PromptReplacement(
91 + modality="image",
92 + target="<image>",
93 + replacement=PromptReplacementDetails(
94 + full="".join([
95 + "<image_bos>",
96 + "<image>" * image_feature_size,
97 + "<image_eos>",
98 + ]),
99 + features="<image>" * image_feature_size,
100 + ),
101 + )
102 +
103 + To avoid unnecessary tokenization during prompt replacement,
104 + we recommended passing token sequences instead of text:
105 +
106 + .. code-block:: python
107 +
108 + PromptReplacement(
109 + modality="image",
110 + target=[image_token_id],
111 + replacement=PromptReplacementDetails(
112 + full=([image_bos_id] + [image_token_id] * image_feature_size
113 + + [image_eos_id]),
114 + features=[image_token_id] * image_feature_size,
115 + ),
116 + )
117 """
118
119 modality: str
120 """The modality for which the replacement is made."""
121
62 - target: _PromptSeq
122 + target: PromptSeq
123 """The token sequence (or text) to find and replace."""
124
65 - replacement: Union[Callable[[int], _PromptRepl],
66 - _PromptRepl] = field(repr=False)
125 + replacement: Union[Callable[[int], PromptRepl],
126 + PromptRepl] = field(repr=False)
127 """
128 Given the index of the processed item within :attr:`modality`,
129 output the replacement token sequence (or text).
@@ -126,6 +186,10 @@ def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
186
187 @dataclass
188 class _BoundPromptSequence:
189 + """
190 + A :data:`_PromptSeq` bound to a tokenizer to automatically
191 + convert between token sequence and text representations.
192 + """
193 tokenizer: AnyTokenizer = field(repr=False)
194
195 _text: Optional[str]
@@ -134,7 +198,7 @@ class _BoundPromptSequence:
198 @staticmethod
199 def from_seq(
200 tokenizer: AnyTokenizer,
137 - seq: _PromptSeq,
201 + seq: PromptSeq,
202 ) -> "_BoundPromptSequence":
203 return _BoundPromptSequence(
204 tokenizer=tokenizer,
@@ -180,9 +244,9 @@ class BoundPromptReplacement:
244 tokenizer: AnyTokenizer = field(repr=False)
245 modality: str
246
183 - _target: _PromptSeq
184 - _replacement: Union[Callable[[int], _PromptRepl],
185 - _PromptRepl] = field(repr=False)
247 + _target: PromptSeq
248 + _replacement: Union[Callable[[int], PromptRepl],
249 + PromptRepl] = field(repr=False)
250
251 def __post_init__(self) -> None:
252 self._replacement_cache = dict[int, _BoundPromptReplacementGroup]()
@@ -350,7 +414,7 @@ def find_text_matches(
414
415
416 def _resolve_matches(
353 - prompt: _PromptSeq,
417 + prompt: PromptSeq,
418 mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
419 ) -> list[_PromptReplacementMatch]:
420 """