A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] [1/N] Initial guide for merged multi-modal processor (#11925)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 10, 2025 at 22:30 UTC
12664ddda522b3a22c5b71eca9b2c907e3a687b3
19 files changed
+403
-168
docs/requirements-docs.txt
+1
@@ -3,6 +3,7 @@ sphinx-book-theme==1.0.1
3
sphinx-copybutton==0.5.2
4
myst-parser==3.0.1
5
sphinx-argparse==0.4.0
6
+sphinx-design==0.6.1
7
sphinx-togglebutton==0.3.2
8
msgspec
9
cloudpickle
docs/source/api/multimodal/index.md
+1
-1
@@ -7,7 +7,7 @@ vLLM provides experimental support for multi-modal models through the {mod}`vllm
7
Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
8
via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
9
10
-Looking to add your own multi-modal model? Please follow the instructions listed [here](#enabling-multimodal-inputs).
10
+Looking to add your own multi-modal model? Please follow the instructions listed [here](#supports-multimodal).
11
12
## Module Contents
13
docs/source/api/multimodal/inputs.md
+1
-1
@@ -3,7 +3,7 @@
3
## User-facing inputs
4
5
```{eval-rst}
6
-.. autodata:: vllm.multimodal.MultiModalDataDict
6
+.. autodata:: vllm.multimodal.inputs.MultiModalDataDict
7
```
8
9
## Internal data structures
docs/source/conf.py
+1
@@ -43,6 +43,7 @@ extensions = [
43
"sphinx.ext.autosummary",
44
"myst_parser",
45
"sphinxarg.ext",
46
+ "sphinx_design",
47
"sphinx_togglebutton",
48
]
49
myst_enable_extensions = [
docs/source/contributing/model/index.md
+1
-1
@@ -2,7 +2,7 @@
2
3
# Adding a New Model
4
5
-This section provides more information on how to integrate a [HuggingFace Transformers](https://github.com/huggingface/transformers) model into vLLM.
5
+This section provides more information on how to integrate a [PyTorch](https://pytorch.org/) model into vLLM.
6
7
```{toctree}
8
:caption: Contents
docs/source/contributing/model/multimodal.md
+316
-64
@@ -1,6 +1,6 @@
1
-(enabling-multimodal-inputs)=
1
+(supports-multimodal)=
2
3
-# Enabling Multimodal Inputs
3
+# Multi-Modal Support
4
5
This document walks you through the steps to extend a basic model so that it accepts [multi-modal inputs](#multimodal-inputs).
6
@@ -37,103 +37,355 @@ Further update the model as follows:
37
) -> SamplerOutput:
38
```
39
40
-## 2. Register input mappers
40
+## 2. Specify processing information
41
42
-For each modality type that the model accepts as input, decorate the model class with {meth}`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
43
-This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in {meth}`~torch.nn.Module.forward`.
42
+Next, create a subclass of {class}`~vllm.multimodal.processing.BaseProcessingInfo`
43
+to provide basic information related to HF processing.
44
45
-```diff
46
- from vllm.model_executor.models.interfaces import SupportsMultiModal
47
-+ from vllm.multimodal import MULTIMODAL_REGISTRY
45
+### Maximum number of input items
46
49
-+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
50
- class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
47
+You need to override the abstract method {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_supported_mm_limits`
48
+to return the maximum number of input items for each modality supported by the model.
49
+
50
+For example, if the model supports any number of images but only one video per prompt:
51
+
52
+```python
53
+def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54
+ return {"image": None, "video": 1}
55
```
56
53
-A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
57
+### Maximum number of placeholder feature tokens
58
+
59
+Also, override the abstract method {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`
60
+to return the maximum number of placeholder feature tokens per input item for each modality.
61
+
62
+When calling the model, the output embeddings from the visual encoder are assigned to the input positions
63
+containing placeholder feature tokens. Therefore, the number of placeholder feature tokens should be equal
64
+to the size of the output embeddings.
65
+
66
+::::{tab-set}
67
+:::{tab-item} Basic example: LLaVA
68
+:sync: llava
69
+
70
+Looking at the code of HF's `LlavaForConditionalGeneration`:
71
+
72
+```python
73
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L530-L544
74
+n_image_tokens = (input_ids == self.config.image_token_index).sum().item()
75
+n_image_features = image_features.shape[0] * image_features.shape[1]
76
+
77
+if n_image_tokens != n_image_features:
78
+ raise ValueError(
79
+ f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}"
80
+ )
81
+special_image_mask = (
82
+ (input_ids == self.config.image_token_index)
83
+ .unsqueeze(-1)
84
+ .expand_as(inputs_embeds)
85
+ .to(inputs_embeds.device)
86
+)
87
+image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
88
+inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
89
+```
90
55
-```{seealso}
56
-[Input Processing Pipeline](#input-processing-pipeline)
91
+The number of placeholder feature tokens per image is `image_features.shape[1]`.
92
+`image_features` is calculated inside the `get_image_features` method:
93
+
94
+```python
95
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L290-L300
96
+image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
97
+
98
+selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
99
+if vision_feature_select_strategy == "default":
100
+ selected_image_feature = selected_image_feature[:, 1:]
101
+elif vision_feature_select_strategy == "full":
102
+ selected_image_feature = selected_image_feature
103
+else:
104
+ raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}")
105
+image_features = self.multi_modal_projector(selected_image_feature)
106
+return image_features
107
```
108
59
-## 3. Register maximum number of multi-modal tokens
109
+We can infer that `image_features.shape[1]` is based on `image_outputs.hidden_states.shape[1]` from the vision tower
110
+(`CLIPVisionModel` for the [`llava-hf/llava-1.5-7b-hf`](https://huggingface.co/llava-hf/llava-1.5-7b-hf) model).
111
+Moreover, we only need the sequence length (the second dimension of the tensor) to get `image_features.shape[1]`.
112
+The sequence length is determined by the initial hidden states in `CLIPVisionTransformer` since the attention
113
+mechanism doesn't change the sequence length of the output hidden states.
114
+
115
+```python
116
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L1094-L1102
117
+hidden_states = self.embeddings(pixel_values, interpolate_pos_encoding=interpolate_pos_encoding)
118
+hidden_states = self.pre_layrnorm(hidden_states)
119
+
120
+encoder_outputs = self.encoder(
121
+ inputs_embeds=hidden_states,
122
+ output_attentions=output_attentions,
123
+ output_hidden_states=output_hidden_states,
124
+ return_dict=return_dict,
125
+)
126
+```
127
61
-For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
62
-and register it via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
128
+To find the sequence length, we turn to the code of `CLIPVisionEmbeddings`:
129
+
130
+```python
131
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L247-L257
132
+target_dtype = self.patch_embedding.weight.dtype
133
+patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid]
134
+patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
135
+
136
+class_embeds = self.class_embedding.expand(batch_size, 1, -1)
137
+embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
138
+if interpolate_pos_encoding:
139
+ embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
140
+else:
141
+ embeddings = embeddings + self.position_embedding(self.position_ids)
142
+return embeddings
143
+```
144
64
-```diff
65
- from vllm.inputs import INPUT_REGISTRY
66
- from vllm.model_executor.models.interfaces import SupportsMultiModal
67
- from vllm.multimodal import MULTIMODAL_REGISTRY
145
+We can infer that `embeddings.shape[1] == self.num_positions`, where
146
69
- @MULTIMODAL_REGISTRY.register_image_input_mapper()
70
-+ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
71
- @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
72
- class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
147
+```python
148
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L195-L196
149
+self.num_patches = (self.image_size // self.patch_size) ** 2
150
+self.num_positions = self.num_patches + 1
151
```
152
75
-Here are some examples:
153
+Overall, the number of placeholder feature tokens for an image can be calculated as:
154
77
-- Image inputs (static feature size): [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
78
-- Image inputs (dynamic feature size): [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
155
+```python
156
+def get_num_image_tokens(
157
+ self,
158
+ *,
159
+ image_width: int,
160
+ image_height: int,
161
+) -> int:
162
+ hf_config = self.get_hf_config()
163
+ hf_processor = self.get_hf_processor()
164
80
-```{seealso}
81
-[Input Processing Pipeline](#input-processing-pipeline)
165
+ image_size = hf_config.vision_config.image_size
166
+ patch_size = hf_config.vision_config.patch_size
167
+
168
+ num_image_tokens = (image_size // patch_size) ** 2 + 1
169
+ if hf_processor.vision_feature_select_strategy == "default":
170
+ num_image_tokens -= 1
171
+
172
+ return num_image_tokens
173
```
174
84
-## 4. (Optional) Register dummy data
175
+Notice that the number of image tokens doesn't depend on the image width and height.
176
+So, we can calculate the maximum number of image tokens using any image size:
177
86
-During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
87
-In such cases, you can define your own dummy data by registering a factory method via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`.
178
+```python
179
+def get_image_size_with_most_features(self) -> ImageSize:
180
+ hf_config = self.get_hf_config()
181
+ width = height = hf_config.image_size
182
+ return ImageSize(width=width, height=height)
183
89
-```diff
90
- from vllm.inputs import INPUT_REGISTRY
91
- from vllm.model_executor.models.interfaces import SupportsMultiModal
92
- from vllm.multimodal import MULTIMODAL_REGISTRY
184
+def get_max_image_tokens(self) -> int:
185
+ target_width, target_height = self.get_image_size_with_most_features()
186
94
- @MULTIMODAL_REGISTRY.register_image_input_mapper()
95
- @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
96
-+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
97
- class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
187
+ return self.get_num_image_tokens(
188
+ image_width=target_width,
189
+ image_height=target_height,
190
+ )
191
+```
192
+
193
+And thus, we can override the method as:
194
+
195
+```python
196
+def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
197
+ return {"image": self.get_max_image_tokens()}
198
```
199
200
```{note}
101
-The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
201
+Our [actual code](gh-file:vllm/model_executor/models/llava.py) is more abstracted to support vision encoders other than CLIP.
202
```
203
+:::
204
+::::
205
+
206
+## 3. Specify dummy inputs
207
+
208
+Then, inherit {class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` to construct dummy inputs for
209
+HF processing as well as memory profiling.
210
+
211
+### For memory profiling
212
+
213
+Override the abstract method {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`
214
+to construct dummy inputs for memory profiling. This dummy input should result in the worst-case memory usage of
215
+the model so that vLLM can reserve the correct amount of memory for it.
216
+
217
+Assuming that the memory usage increases with the number of tokens, the dummy input can be constructed based
218
+on the code for {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`.
219
+
220
+::::{tab-set}
221
+:::{tab-item} Basic example: LLaVA
222
+:sync: llava
223
+Making use of the `get_image_size_with_most_features` method implemented in the previous section:
224
+
225
+```python
226
+def get_dummy_processor_inputs(
227
+ self,
228
+ seq_len: int,
229
+ mm_counts: Mapping[str, int],
230
+) -> ProcessorInputs:
231
+ num_images = mm_counts.get("image", 0)
232
+
233
+ processor = self.info.get_hf_processor()
234
+ image_token = processor.image_token
235
+
236
+ hf_config = self.get_hf_config()
237
+ target_width, target_height = self.info.get_image_size_with_most_features()
238
+
239
+ mm_data = {
240
+ "image":
241
+ self._get_dummy_images(width=target_width,
242
+ height=target_height,
243
+ num_images=num_images)
244
+ }
245
+
246
+ return ProcessorInputs(
247
+ prompt_text=image_token * num_images,
248
+ mm_data=mm_data,
249
+ )
250
+```
251
+:::
252
+::::
253
104
-Here are some examples:
254
+## 4. Specify processing details
255
106
-- Image inputs (static feature size): [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
107
-- Image inputs (dynamic feature size): [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
256
+Afterwards, create a subclass of {class}`~vllm.multimodal.processing.BaseMultiModalProcessor`
257
+to fill in the missing details about HF processing.
258
259
```{seealso}
110
-[Input Processing Pipeline](#input-processing-pipeline)
260
+[Multi-Modal Data Processing](#mm-processing)
261
```
262
113
-## 5. (Optional) Register input processor
263
+### Multi-modal fields
264
+
265
+Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` to
266
+return a schema of the tensors outputted by the HF processor that are related to the input multi-modal items.
267
+
268
+::::{tab-set}
269
+:::{tab-item} Basic example: LLaVA
270
+:sync: llava
271
+
272
+Looking at the model's `forward` method:
273
+
274
+```python
275
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L387-L404
276
+def forward(
277
+ self,
278
+ input_ids: torch.LongTensor = None,
279
+ pixel_values: torch.FloatTensor = None,
280
+ attention_mask: Optional[torch.Tensor] = None,
281
+ position_ids: Optional[torch.LongTensor] = None,
282
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
283
+ inputs_embeds: Optional[torch.FloatTensor] = None,
284
+ vision_feature_layer: Optional[int] = None,
285
+ vision_feature_select_strategy: Optional[str] = None,
286
+ labels: Optional[torch.LongTensor] = None,
287
+ use_cache: Optional[bool] = None,
288
+ output_attentions: Optional[bool] = None,
289
+ output_hidden_states: Optional[bool] = None,
290
+ return_dict: Optional[bool] = None,
291
+ cache_position: Optional[torch.LongTensor] = None,
292
+ num_logits_to_keep: int = 0,
293
+) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
294
+```
295
115
-Sometimes, there is a need to process inputs at the {class}`~vllm.LLMEngine` level before they are passed to the model executor.
116
-This is often due to the fact that unlike implementations in HuggingFace Transformers, the reshaping and/or expansion of multi-modal embeddings needs to take place outside model's {meth}`~torch.nn.Module.forward` call.
117
-You can register input processors via {meth}`INPUT_REGISTRY.register_input_processor <vllm.inputs.registry.InputRegistry.register_input_processor>`.
296
+The only related keyword argument is `pixel_values` which directly corresponds to input images.
297
+The shape of `pixel_values` is `(N, C, H, W)` where `N` is the number of images.
298
+So, we override the method as follows:
299
+
300
+```python
301
+def _get_mm_fields_config(
302
+ self,
303
+ hf_inputs: BatchFeature,
304
+ hf_processor_mm_kwargs: Mapping[str, object],
305
+) -> Mapping[str, MultiModalFieldConfig]:
306
+ return dict(
307
+ pixel_values=MultiModalFieldConfig.batched("image"),
308
+ )
309
+```
310
119
-```diff
120
- from vllm.inputs import INPUT_REGISTRY
121
- from vllm.model_executor.models.interfaces import SupportsMultiModal
122
- from vllm.multimodal import MULTIMODAL_REGISTRY
311
+```{note}
312
+Our [actual code](gh-file:vllm/model_executor/models/llava.py) additionally supports
313
+pre-computed image embeddings, which can be passed to be model via the `image_embeds` argument.
314
+```
315
+:::
316
+::::
317
124
- @MULTIMODAL_REGISTRY.register_image_input_mapper()
125
- @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
126
- @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
127
-+ @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
128
- class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
318
+### Prompt replacements
319
+
320
+Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` to
321
+return a list of {class}`~vllm.multimodal.processing.PromptReplacement` instances.
322
+
323
+Each {class}`~vllm.multimodal.processing.PromptReplacement` instance specifies a find-and-replace
324
+operation performed by the HF processor.
325
+
326
+::::{tab-set}
327
+:::{tab-item} Basic example: LLaVA
328
+:sync: llava
329
+
330
+Looking at HF's `LlavaProcessor`:
331
+
332
+```python
333
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/processing_llava.py#L167-L170
334
+prompt_strings = []
335
+for sample in text:
336
+ sample = sample.replace(self.image_token, self.image_token * num_image_tokens)
337
+ prompt_strings.append(sample)
338
```
339
131
-A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
132
-Here are some examples:
340
+It simply repeats each input `image_token` a number of times equal to the number of placeholder feature tokens (`num_image_tokens`).
341
+Based on this, we override the method as follows:
342
+
343
+```python
344
+def _get_prompt_replacements(
345
+ self,
346
+ mm_items: MultiModalDataItems,
347
+ hf_processor_mm_kwargs: Mapping[str, object],
348
+ out_mm_kwargs: MultiModalKwargs,
349
+) -> list[PromptReplacement]:
350
+ hf_config = self.info.get_hf_config()
351
+ image_token_id = hf_config.image_token_index
352
+
353
+ def get_replacement(item_idx: int):
354
+ images = mm_items.get_items("image", ImageProcessorItems)
355
+
356
+ image_size = images.get_image_size(item_idx)
357
+ num_image_tokens = self.info.get_num_image_tokens(
358
+ image_width=image_size.width,
359
+ image_height=image_size.height,
360
+ )
361
+
362
+ return [image_token_id] * num_image_tokens
363
+
364
+ return [
365
+ PromptReplacement(
366
+ modality="image",
367
+ target=[image_token_id],
368
+ replacement=get_replacement,
369
+ ),
370
+ ]
371
+```
372
+:::
373
+::::
374
134
-- Insert static number of image tokens: [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
135
-- Insert dynamic number of image tokens: [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
375
+## 5. Register processor-related classes
376
137
-```{seealso}
138
-[Input Processing Pipeline](#input-processing-pipeline)
377
+After you have defined {class}`~vllm.multimodal.processing.BaseProcessingInfo` (Step 2),
378
+{class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` (Step 3),
379
+and {class}`~vllm.multimodal.processing.BaseMultiModalProcessor` (Step 4),
380
+decorate the model class with {meth}`MULTIMODAL_REGISTRY.register_processor <vllm.multimodal.registry.MultiModalRegistry.register_processor>`
381
+to register them to the multi-modal registry:
382
+
383
+```diff
384
+ from vllm.model_executor.models.interfaces import SupportsMultiModal
385
++ from vllm.multimodal import MULTIMODAL_REGISTRY
386
+
387
++ @MULTIMODAL_REGISTRY.register_processor(YourMultiModalProcessor,
388
++ info=YourProcessingInfo,
389
++ dummy_inputs=YourDummyInputsBuilder)
390
+ class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
391
```
docs/source/contributing/model/registration.md
+1
-1
@@ -48,7 +48,7 @@ ModelRegistry.register_model("YourModelForCausalLM", "your_code:YourModelForCaus
48
49
```{important}
50
If your model is a multimodal model, ensure the model class implements the {class}`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
51
-Read more about that [here](#enabling-multimodal-inputs).
51
+Read more about that [here](#supports-multimodal).
52
```
53
54
```{note}
docs/source/design/input_processing/input_processing_pipeline.md
deleted
-19
@@ -1,19 +0,0 @@
1
-(input-processing-pipeline)=
2
-
3
-# Input Processing Pipeline
4
-
5
-1. Input data is passed to {class}`~vllm.LLMEngine` (or {class}`~vllm.AsyncLLMEngine`).
6
-
7
-2. Tokenize the data if necessary.
8
-
9
-3. Process the inputs using {meth}`INPUT_REGISTRY.process_input <vllm.inputs.registry.InputRegistry.process_input>`.
10
-
11
- - For example, add placeholder tokens to reserve KV cache for multi-modal embeddings.
12
-
13
-4. Send the processed inputs to {class}`~vllm.executor.executor_base.ExecutorBase`.
14
-
15
-5. Distribute the inputs via {class}`~vllm.worker.worker_base.WorkerBase` to {class}`~vllm.worker.model_runner_base.ModelRunnerBase`.
16
-
17
-6. If the data contains multi-modal data, convert it into keyword arguments using {meth}`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
18
-
19
- - For example, convert a {class}`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/design/input_processing/model_inputs_index.md
deleted
-43
@@ -1,43 +0,0 @@
1
-(input-processing)=
2
-
3
-# Input Processing
4
-
5
-```{eval-rst}
6
-.. currentmodule:: vllm.inputs
7
-```
8
-
9
-Each model can override parts of vLLM's [input processing pipeline](#input-processing-pipeline) via
10
-{data}`~vllm.inputs.INPUT_REGISTRY` and {data}`~vllm.multimodal.MULTIMODAL_REGISTRY`.
11
-
12
-Currently, this mechanism is only utilized in [multi-modal](#multi-modality) models for preprocessing multi-modal input
13
-data in addition to input prompt, but it can be extended to text-only language models when needed.
14
-
15
-## Guides
16
-
17
-```{toctree}
18
-:maxdepth: 1
19
-
20
-input_processing_pipeline
21
-```
22
-
23
-## Module Contents
24
-
25
-### LLM Engine Inputs
26
-
27
-```{eval-rst}
28
-.. autoclass:: vllm.inputs.DecoderOnlyInputs
29
- :members:
30
- :show-inheritance:
31
-```
32
-
33
-### Registry
34
-
35
-```{eval-rst}
36
-.. autodata:: vllm.inputs.INPUT_REGISTRY
37
-```
38
-
39
-```{eval-rst}
40
-.. automodule:: vllm.inputs.registry
41
- :members:
42
- :show-inheritance:
43
-```
docs/source/design/mm_processing.md
new
+64
@@ -0,0 +1,64 @@
1
+(mm-processing)=
2
+
3
+# Multi-Modal Data Processing
4
+
5
+To enable various optimizations in vLLM such as [chunked prefill](#chunked-prefill) and [prefix caching](#automatic-prefix-caching), we use {class}`~vllm.multimodal.processing.BaseMultiModalProcessor` to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
6
+
7
+Here are the main features of {class}`~vllm.multimodal.processing.BaseMultiModalProcessor`:
8
+
9
+## Prompt Replacement Detection
10
+
11
+One of the main responsibilies of HF processor is to replace input placeholder tokens (e.g. `<image>` for a single image) with feature placeholder tokens (e.g. `<image><image>...<image>`, the number of which equals to the feature size). The information about which tokens have been replaced is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
12
+
13
+In vLLM, this information is specified using {class}`~vllm.multimodal.processing.PromptReplacement` in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`. Given this specification, we can automatically detect whether HF has replaced the input placeholder tokens by checking whether the feature placeholder tokens exist in the prompt.
14
+
15
+## Tokenized Prompt Inputs
16
+
17
+To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data.
18
+
19
+### The problem
20
+
21
+Consider that HF processors follow these main steps:
22
+
23
+1. Tokenize the text
24
+2. Process multi-modal inputs
25
+3. Perform prompt replacement
26
+
27
+And we require that:
28
+
29
+- For text + multi-modal inputs, apply all steps 1--3.
30
+- For tokenized + multi-modal inputs, apply only steps 2--3.
31
+
32
+How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs:
33
+
34
+- For text + multi-modal inputs, simply call the HF processor directly.
35
+- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs.
36
+
37
+While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs.
38
+
39
+Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other.
40
+
41
+(mm-dummy-text)=
42
+
43
+### Dummy text
44
+
45
+We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
46
+
47
+(mm-automatic-prompt-replacement)=
48
+
49
+### Automatic prompt replacement
50
+
51
+We address the second issue by implementing model-agnostic code in
52
+{meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_replacements` to automatically replace input placeholder tokens with feature placeholder tokens based on the specification outputted by {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`.
53
+
54
+### Summary
55
+
56
+With the help of dummy text and automatic prompt replacement, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main`.
57
+
58
+## Processor Output Caching
59
+
60
+Some HF processors, such as the one for Qwen2-VL, are [very slow](gh-issue:9238). To alleviate this problem, we cache the multi-modal outputs of HF processor to avoid processing the same multi-modal input (e.g. image) again.
61
+
62
+When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
63
+
64
+Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#mm-dummy-text) to avoid HF errors. Since this skips HF's prompt replacement code, we apply [automatic prompt replacement](#mm-automatic-prompt-replacement) afterwards to keep the output tokens and multi-modal data consistent with each other.
docs/source/index.md
+1
-1
@@ -145,7 +145,7 @@ design/arch_overview
145
design/huggingface_integration
146
design/plugin_system
147
design/kernel/paged_attention
148
-design/input_processing/model_inputs_index
148
+design/mm_processing
149
design/automatic_prefix_caching
150
design/multiprocessing
151
```
docs/source/serving/multimodal_inputs.md
+1
-1
@@ -14,7 +14,7 @@ and [open an issue on GitHub](https://github.com/vllm-project/vllm/issues/new/ch
14
To input multi-modal data, follow this schema in {class}`vllm.inputs.PromptType`:
15
16
- `prompt`: The prompt should follow the format that is documented on HuggingFace.
17
-- `multi_modal_data`: This is a dictionary that follows the schema defined in {class}`vllm.multimodal.MultiModalDataDict`.
17
+- `multi_modal_data`: This is a dictionary that follows the schema defined in {class}`vllm.multimodal.inputs.MultiModalDataDict`.
18
19
### Image
20
vllm/config.py
+1
-2
@@ -2124,8 +2124,7 @@ class MultiModalConfig:
2124
2125
limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
2126
"""
2127
- The maximum number of multi-modal input instances allowed per prompt
2128
- for each :class:`~vllm.multimodal.MultiModalPlugin`.
2127
+ The maximum number of input items allowed per prompt for each modality.
2128
"""
2129
2130
def compute_hash(self) -> str:
vllm/inputs/__init__.py
-3
@@ -11,9 +11,6 @@ INPUT_REGISTRY = InputRegistry()
11
"""
12
The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
13
to dispatch data processing according to the target model.
14
-
15
-See also:
16
- :ref:`input-processing-pipeline`
14
"""
15
16
__all__ = [
vllm/inputs/registry.py
+2
-10
@@ -313,9 +313,6 @@ class InputRegistry:
313
314
The model is identified by ``model_config``.
315
316
- See also:
317
- :ref:`enabling-multimodal-inputs`
318
-
316
Note:
317
This should be called after
318
:meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
@@ -384,10 +381,8 @@ class InputRegistry:
381
Register an input processor to a model class.
382
383
The provided function is invoked on each input to the model. This
387
- happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
388
-
389
- See also:
390
- :ref:`input-processing-pipeline`
384
+ happens before
385
+ :meth:`~vllm.multimodal.registry.MultiModalRegistry.map_input`.
386
"""
387
388
def wrapper(model_cls: N) -> N:
@@ -429,9 +424,6 @@ class InputRegistry:
424
Apply an input processor to an instance of model inputs.
425
426
The model is identified by ``model_config``.
432
-
433
- See also:
434
- :ref:`input-processing-pipeline`
427
"""
428
# Avoid circular import
429
from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/__init__.py
+2
-2
@@ -8,10 +8,10 @@ from .registry import MultiModalRegistry
8
MULTIMODAL_REGISTRY = MultiModalRegistry()
9
"""
10
The global :class:`~MultiModalRegistry` is used by model runners to
11
-dispatch data processing according to its modality and the target model.
11
+dispatch data processing according to the target model.
12
13
See also:
14
- :ref:`input-processing-pipeline`
14
+ :ref:`mm-processing`
15
"""
16
17
__all__ = [
vllm/multimodal/base.py
-14
@@ -90,10 +90,6 @@ class MultiModalPlugin(ABC):
90
invoked to transform the data into a dictionary of model inputs.
91
92
If `None` is provided, then the default input mapper is used instead.
93
-
94
- See also:
95
- - :ref:`input-processing-pipeline`
96
- - :ref:`enabling-multimodal-inputs`
93
"""
94
95
def wrapper(model_cls: N) -> N:
@@ -126,10 +122,6 @@ class MultiModalPlugin(ABC):
122
123
Raises:
124
TypeError: If the data type is not supported.
129
-
130
- See also:
131
- - :ref:`input-processing-pipeline`
132
- - :ref:`enabling-multimodal-inputs`
125
"""
126
127
# Avoid circular import
@@ -186,9 +178,6 @@ class MultiModalPlugin(ABC):
178
for a model class.
179
180
If `None` is provided, then the default calculation is used instead.
189
-
190
- See also:
191
- :ref:`enabling-multimodal-inputs`
181
"""
182
183
def wrapper(model_cls: N) -> N:
@@ -218,9 +207,6 @@ class MultiModalPlugin(ABC):
207
If this registry is not applicable to the model, `0` is returned.
208
209
The model is identified by ``model_config``.
221
-
222
- See also:
223
- :ref:`enabling-multimodal-inputs`
210
"""
211
# Avoid circular import
212
from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/inputs.py
+2
-1
@@ -493,7 +493,8 @@ A dictionary containing placeholder ranges for each modality.
493
494
class MultiModalInputsV2(TypedDict):
495
"""
496
- Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
496
+ Represents the outputs of
497
+ :class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
498
ready to be passed to vLLM internals.
499
"""
500
vllm/multimodal/registry.py
+8
-4
@@ -100,8 +100,7 @@ class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
100
101
class MultiModalRegistry:
102
"""
103
- A registry that dispatches data processing to the
104
- :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
103
+ A registry that dispatches data processing according to the model.
104
"""
105
106
DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
@@ -367,8 +366,7 @@ class MultiModalRegistry:
366
invoked to transform the data into a dictionary of model inputs.
367
368
See also:
370
- - :ref:`input-processing-pipeline`
371
- - :ref:`enabling-multimodal-inputs`
369
+ :ref:`mm-processing`
370
"""
371
372
def wrapper(model_cls: N) -> N:
@@ -398,6 +396,9 @@ class MultiModalRegistry:
396
def has_processor(self, model_config: "ModelConfig") -> bool:
397
"""
398
Test whether a multi-modal processor is defined for a specific model.
399
+
400
+ See also:
401
+ :ref:`mm-processing`
402
"""
403
return self._get_model_cls(model_config) in self._processor_factories
404
@@ -408,6 +409,9 @@ class MultiModalRegistry:
409
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
410
"""
411
Create a multi-modal processor for a specific model and tokenizer.
412
+
413
+ See also:
414
+ :ref:`mm-processing`
415
"""
416
model_cls = self._get_model_cls(model_config)
417
factories = self._processor_factories[model_cls]