@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 12664ddda5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] [1/N] Initial guide for merged multi-modal processor (#11925)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 10, 2025 at 22:30 UTC 12664ddda522b3a22c5b71eca9b2c907e3a687b3
19 files changed +403 -168
docs/requirements-docs.txt
+1
@@ -3,6 +3,7 @@ sphinx-book-theme==1.0.1
3 sphinx-copybutton==0.5.2
4 myst-parser==3.0.1
5 sphinx-argparse==0.4.0
6 +sphinx-design==0.6.1
7 sphinx-togglebutton==0.3.2
8 msgspec
9 cloudpickle
docs/source/api/multimodal/index.md
+1 -1
@@ -7,7 +7,7 @@ vLLM provides experimental support for multi-modal models through the {mod}`vllm
7 Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
8 via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
9
10 -Looking to add your own multi-modal model? Please follow the instructions listed [here](#enabling-multimodal-inputs).
10 +Looking to add your own multi-modal model? Please follow the instructions listed [here](#supports-multimodal).
11
12 ## Module Contents
13
docs/source/api/multimodal/inputs.md
+1 -1
@@ -3,7 +3,7 @@
3 ## User-facing inputs
4
5 ```{eval-rst}
6 -.. autodata:: vllm.multimodal.MultiModalDataDict
6 +.. autodata:: vllm.multimodal.inputs.MultiModalDataDict
7 ```
8
9 ## Internal data structures
docs/source/conf.py
+1
@@ -43,6 +43,7 @@ extensions = [
43 "sphinx.ext.autosummary",
44 "myst_parser",
45 "sphinxarg.ext",
46 + "sphinx_design",
47 "sphinx_togglebutton",
48 ]
49 myst_enable_extensions = [
docs/source/contributing/model/index.md
+1 -1
@@ -2,7 +2,7 @@
2
3 # Adding a New Model
4
5 -This section provides more information on how to integrate a [HuggingFace Transformers](https://github.com/huggingface/transformers) model into vLLM.
5 +This section provides more information on how to integrate a [PyTorch](https://pytorch.org/) model into vLLM.
6
7 ```{toctree}
8 :caption: Contents
docs/source/contributing/model/multimodal.md
+316 -64
@@ -1,6 +1,6 @@
1 -(enabling-multimodal-inputs)=
1 +(supports-multimodal)=
2
3 -# Enabling Multimodal Inputs
3 +# Multi-Modal Support
4
5 This document walks you through the steps to extend a basic model so that it accepts [multi-modal inputs](#multimodal-inputs).
6
@@ -37,103 +37,355 @@ Further update the model as follows:
37 ) -> SamplerOutput:
38 ```
39
40 -## 2. Register input mappers
40 +## 2. Specify processing information
41
42 -For each modality type that the model accepts as input, decorate the model class with {meth}`MULTIMODAL_REGISTRY.register_input_mapper <vllm.multimodal.MultiModalRegistry.register_input_mapper>`.
43 -This decorator accepts a function that maps multi-modal inputs to the keyword arguments you have previously defined in {meth}`~torch.nn.Module.forward`.
42 +Next, create a subclass of {class}`~vllm.multimodal.processing.BaseProcessingInfo`
43 +to provide basic information related to HF processing.
44
45 -```diff
46 - from vllm.model_executor.models.interfaces import SupportsMultiModal
47 -+ from vllm.multimodal import MULTIMODAL_REGISTRY
45 +### Maximum number of input items
46
49 -+ @MULTIMODAL_REGISTRY.register_image_input_mapper()
50 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
47 +You need to override the abstract method {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_supported_mm_limits`
48 +to return the maximum number of input items for each modality supported by the model.
49 +
50 +For example, if the model supports any number of images but only one video per prompt:
51 +
52 +```python
53 +def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54 + return {"image": None, "video": 1}
55 ```
56
53 -A default mapper is available for each modality in the core vLLM library. This input mapper will be used if you do not provide your own function.
57 +### Maximum number of placeholder feature tokens
58 +
59 +Also, override the abstract method {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`
60 +to return the maximum number of placeholder feature tokens per input item for each modality.
61 +
62 +When calling the model, the output embeddings from the visual encoder are assigned to the input positions
63 +containing placeholder feature tokens. Therefore, the number of placeholder feature tokens should be equal
64 +to the size of the output embeddings.
65 +
66 +::::{tab-set}
67 +:::{tab-item} Basic example: LLaVA
68 +:sync: llava
69 +
70 +Looking at the code of HF's `LlavaForConditionalGeneration`:
71 +
72 +```python
73 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L530-L544
74 +n_image_tokens = (input_ids == self.config.image_token_index).sum().item()
75 +n_image_features = image_features.shape[0] * image_features.shape[1]
76 +
77 +if n_image_tokens != n_image_features:
78 + raise ValueError(
79 + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}"
80 + )
81 +special_image_mask = (
82 + (input_ids == self.config.image_token_index)
83 + .unsqueeze(-1)
84 + .expand_as(inputs_embeds)
85 + .to(inputs_embeds.device)
86 +)
87 +image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
88 +inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
89 +```
90
55 -```{seealso}
56 -[Input Processing Pipeline](#input-processing-pipeline)
91 +The number of placeholder feature tokens per image is `image_features.shape[1]`.
92 +`image_features` is calculated inside the `get_image_features` method:
93 +
94 +```python
95 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L290-L300
96 +image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
97 +
98 +selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
99 +if vision_feature_select_strategy == "default":
100 + selected_image_feature = selected_image_feature[:, 1:]
101 +elif vision_feature_select_strategy == "full":
102 + selected_image_feature = selected_image_feature
103 +else:
104 + raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}")
105 +image_features = self.multi_modal_projector(selected_image_feature)
106 +return image_features
107 ```
108
59 -## 3. Register maximum number of multi-modal tokens
109 +We can infer that `image_features.shape[1]` is based on `image_outputs.hidden_states.shape[1]` from the vision tower
110 +(`CLIPVisionModel` for the [`llava-hf/llava-1.5-7b-hf`](https://huggingface.co/llava-hf/llava-1.5-7b-hf) model).
111 +Moreover, we only need the sequence length (the second dimension of the tensor) to get `image_features.shape[1]`.
112 +The sequence length is determined by the initial hidden states in `CLIPVisionTransformer` since the attention
113 +mechanism doesn't change the sequence length of the output hidden states.
114 +
115 +```python
116 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L1094-L1102
117 +hidden_states = self.embeddings(pixel_values, interpolate_pos_encoding=interpolate_pos_encoding)
118 +hidden_states = self.pre_layrnorm(hidden_states)
119 +
120 +encoder_outputs = self.encoder(
121 + inputs_embeds=hidden_states,
122 + output_attentions=output_attentions,
123 + output_hidden_states=output_hidden_states,
124 + return_dict=return_dict,
125 +)
126 +```
127
61 -For each modality type that the model accepts as input, calculate the maximum possible number of tokens per data item
62 -and register it via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_max_multimodal_tokens>`.
128 +To find the sequence length, we turn to the code of `CLIPVisionEmbeddings`:
129 +
130 +```python
131 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L247-L257
132 +target_dtype = self.patch_embedding.weight.dtype
133 +patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid]
134 +patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
135 +
136 +class_embeds = self.class_embedding.expand(batch_size, 1, -1)
137 +embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
138 +if interpolate_pos_encoding:
139 + embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
140 +else:
141 + embeddings = embeddings + self.position_embedding(self.position_ids)
142 +return embeddings
143 +```
144
64 -```diff
65 - from vllm.inputs import INPUT_REGISTRY
66 - from vllm.model_executor.models.interfaces import SupportsMultiModal
67 - from vllm.multimodal import MULTIMODAL_REGISTRY
145 +We can infer that `embeddings.shape[1] == self.num_positions`, where
146
69 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
70 -+ @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
71 - @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
72 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
147 +```python
148 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/modeling_clip.py#L195-L196
149 +self.num_patches = (self.image_size // self.patch_size) ** 2
150 +self.num_positions = self.num_patches + 1
151 ```
152
75 -Here are some examples:
153 +Overall, the number of placeholder feature tokens for an image can be calculated as:
154
77 -- Image inputs (static feature size): [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
78 -- Image inputs (dynamic feature size): [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
155 +```python
156 +def get_num_image_tokens(
157 + self,
158 + *,
159 + image_width: int,
160 + image_height: int,
161 +) -> int:
162 + hf_config = self.get_hf_config()
163 + hf_processor = self.get_hf_processor()
164
80 -```{seealso}
81 -[Input Processing Pipeline](#input-processing-pipeline)
165 + image_size = hf_config.vision_config.image_size
166 + patch_size = hf_config.vision_config.patch_size
167 +
168 + num_image_tokens = (image_size // patch_size) ** 2 + 1
169 + if hf_processor.vision_feature_select_strategy == "default":
170 + num_image_tokens -= 1
171 +
172 + return num_image_tokens
173 ```
174
84 -## 4. (Optional) Register dummy data
175 +Notice that the number of image tokens doesn't depend on the image width and height.
176 +So, we can calculate the maximum number of image tokens using any image size:
177
86 -During startup, dummy data is passed to the vLLM model to allocate memory. This only consists of text input by default, which may not be applicable to multi-modal models.
87 -In such cases, you can define your own dummy data by registering a factory method via {meth}`INPUT_REGISTRY.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`.
178 +```python
179 +def get_image_size_with_most_features(self) -> ImageSize:
180 + hf_config = self.get_hf_config()
181 + width = height = hf_config.image_size
182 + return ImageSize(width=width, height=height)
183
89 -```diff
90 - from vllm.inputs import INPUT_REGISTRY
91 - from vllm.model_executor.models.interfaces import SupportsMultiModal
92 - from vllm.multimodal import MULTIMODAL_REGISTRY
184 +def get_max_image_tokens(self) -> int:
185 + target_width, target_height = self.get_image_size_with_most_features()
186
94 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
95 - @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
96 -+ @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
97 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
187 + return self.get_num_image_tokens(
188 + image_width=target_width,
189 + image_height=target_height,
190 + )
191 +```
192 +
193 +And thus, we can override the method as:
194 +
195 +```python
196 +def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
197 + return {"image": self.get_max_image_tokens()}
198 ```
199
200 ```{note}
101 -The dummy data should have the maximum possible number of multi-modal tokens, as described in the previous step.
201 +Our [actual code](gh-file:vllm/model_executor/models/llava.py) is more abstracted to support vision encoders other than CLIP.
202 ```
203 +:::
204 +::::
205 +
206 +## 3. Specify dummy inputs
207 +
208 +Then, inherit {class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` to construct dummy inputs for
209 +HF processing as well as memory profiling.
210 +
211 +### For memory profiling
212 +
213 +Override the abstract method {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`
214 +to construct dummy inputs for memory profiling. This dummy input should result in the worst-case memory usage of
215 +the model so that vLLM can reserve the correct amount of memory for it.
216 +
217 +Assuming that the memory usage increases with the number of tokens, the dummy input can be constructed based
218 +on the code for {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`.
219 +
220 +::::{tab-set}
221 +:::{tab-item} Basic example: LLaVA
222 +:sync: llava
223 +Making use of the `get_image_size_with_most_features` method implemented in the previous section:
224 +
225 +```python
226 +def get_dummy_processor_inputs(
227 + self,
228 + seq_len: int,
229 + mm_counts: Mapping[str, int],
230 +) -> ProcessorInputs:
231 + num_images = mm_counts.get("image", 0)
232 +
233 + processor = self.info.get_hf_processor()
234 + image_token = processor.image_token
235 +
236 + hf_config = self.get_hf_config()
237 + target_width, target_height = self.info.get_image_size_with_most_features()
238 +
239 + mm_data = {
240 + "image":
241 + self._get_dummy_images(width=target_width,
242 + height=target_height,
243 + num_images=num_images)
244 + }
245 +
246 + return ProcessorInputs(
247 + prompt_text=image_token * num_images,
248 + mm_data=mm_data,
249 + )
250 +```
251 +:::
252 +::::
253
104 -Here are some examples:
254 +## 4. Specify processing details
255
106 -- Image inputs (static feature size): [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
107 -- Image inputs (dynamic feature size): [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
256 +Afterwards, create a subclass of {class}`~vllm.multimodal.processing.BaseMultiModalProcessor`
257 +to fill in the missing details about HF processing.
258
259 ```{seealso}
110 -[Input Processing Pipeline](#input-processing-pipeline)
260 +[Multi-Modal Data Processing](#mm-processing)
261 ```
262
113 -## 5. (Optional) Register input processor
263 +### Multi-modal fields
264 +
265 +Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` to
266 +return a schema of the tensors outputted by the HF processor that are related to the input multi-modal items.
267 +
268 +::::{tab-set}
269 +:::{tab-item} Basic example: LLaVA
270 +:sync: llava
271 +
272 +Looking at the model's `forward` method:
273 +
274 +```python
275 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L387-L404
276 +def forward(
277 + self,
278 + input_ids: torch.LongTensor = None,
279 + pixel_values: torch.FloatTensor = None,
280 + attention_mask: Optional[torch.Tensor] = None,
281 + position_ids: Optional[torch.LongTensor] = None,
282 + past_key_values: Optional[List[torch.FloatTensor]] = None,
283 + inputs_embeds: Optional[torch.FloatTensor] = None,
284 + vision_feature_layer: Optional[int] = None,
285 + vision_feature_select_strategy: Optional[str] = None,
286 + labels: Optional[torch.LongTensor] = None,
287 + use_cache: Optional[bool] = None,
288 + output_attentions: Optional[bool] = None,
289 + output_hidden_states: Optional[bool] = None,
290 + return_dict: Optional[bool] = None,
291 + cache_position: Optional[torch.LongTensor] = None,
292 + num_logits_to_keep: int = 0,
293 +) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
294 +```
295
115 -Sometimes, there is a need to process inputs at the {class}`~vllm.LLMEngine` level before they are passed to the model executor.
116 -This is often due to the fact that unlike implementations in HuggingFace Transformers, the reshaping and/or expansion of multi-modal embeddings needs to take place outside model's {meth}`~torch.nn.Module.forward` call.
117 -You can register input processors via {meth}`INPUT_REGISTRY.register_input_processor <vllm.inputs.registry.InputRegistry.register_input_processor>`.
296 +The only related keyword argument is `pixel_values` which directly corresponds to input images.
297 +The shape of `pixel_values` is `(N, C, H, W)` where `N` is the number of images.
298 +So, we override the method as follows:
299 +
300 +```python
301 +def _get_mm_fields_config(
302 + self,
303 + hf_inputs: BatchFeature,
304 + hf_processor_mm_kwargs: Mapping[str, object],
305 +) -> Mapping[str, MultiModalFieldConfig]:
306 + return dict(
307 + pixel_values=MultiModalFieldConfig.batched("image"),
308 + )
309 +```
310
119 -```diff
120 - from vllm.inputs import INPUT_REGISTRY
121 - from vllm.model_executor.models.interfaces import SupportsMultiModal
122 - from vllm.multimodal import MULTIMODAL_REGISTRY
311 +```{note}
312 +Our [actual code](gh-file:vllm/model_executor/models/llava.py) additionally supports
313 +pre-computed image embeddings, which can be passed to be model via the `image_embeds` argument.
314 +```
315 +:::
316 +::::
317
124 - @MULTIMODAL_REGISTRY.register_image_input_mapper()
125 - @MULTIMODAL_REGISTRY.register_max_image_tokens(<your_calculation>)
126 - @INPUT_REGISTRY.register_dummy_data(<your_dummy_data_factory>)
127 -+ @INPUT_REGISTRY.register_input_processor(<your_input_processor>)
128 - class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
318 +### Prompt replacements
319 +
320 +Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` to
321 +return a list of {class}`~vllm.multimodal.processing.PromptReplacement` instances.
322 +
323 +Each {class}`~vllm.multimodal.processing.PromptReplacement` instance specifies a find-and-replace
324 +operation performed by the HF processor.
325 +
326 +::::{tab-set}
327 +:::{tab-item} Basic example: LLaVA
328 +:sync: llava
329 +
330 +Looking at HF's `LlavaProcessor`:
331 +
332 +```python
333 +# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/processing_llava.py#L167-L170
334 +prompt_strings = []
335 +for sample in text:
336 + sample = sample.replace(self.image_token, self.image_token * num_image_tokens)
337 + prompt_strings.append(sample)
338 ```
339
131 -A common use case of input processors is inserting placeholder tokens to leverage the vLLM framework for attention mask generation.
132 -Here are some examples:
340 +It simply repeats each input `image_token` a number of times equal to the number of placeholder feature tokens (`num_image_tokens`).
341 +Based on this, we override the method as follows:
342 +
343 +```python
344 +def _get_prompt_replacements(
345 + self,
346 + mm_items: MultiModalDataItems,
347 + hf_processor_mm_kwargs: Mapping[str, object],
348 + out_mm_kwargs: MultiModalKwargs,
349 +) -> list[PromptReplacement]:
350 + hf_config = self.info.get_hf_config()
351 + image_token_id = hf_config.image_token_index
352 +
353 + def get_replacement(item_idx: int):
354 + images = mm_items.get_items("image", ImageProcessorItems)
355 +
356 + image_size = images.get_image_size(item_idx)
357 + num_image_tokens = self.info.get_num_image_tokens(
358 + image_width=image_size.width,
359 + image_height=image_size.height,
360 + )
361 +
362 + return [image_token_id] * num_image_tokens
363 +
364 + return [
365 + PromptReplacement(
366 + modality="image",
367 + target=[image_token_id],
368 + replacement=get_replacement,
369 + ),
370 + ]
371 +```
372 +:::
373 +::::
374
134 -- Insert static number of image tokens: [LLaVA-1.5 Model](gh-file:vllm/model_executor/models/llava.py)
135 -- Insert dynamic number of image tokens: [LLaVA-NeXT Model](gh-file:vllm/model_executor/models/llava_next.py)
375 +## 5. Register processor-related classes
376
137 -```{seealso}
138 -[Input Processing Pipeline](#input-processing-pipeline)
377 +After you have defined {class}`~vllm.multimodal.processing.BaseProcessingInfo` (Step 2),
378 +{class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` (Step 3),
379 +and {class}`~vllm.multimodal.processing.BaseMultiModalProcessor` (Step 4),
380 +decorate the model class with {meth}`MULTIMODAL_REGISTRY.register_processor <vllm.multimodal.registry.MultiModalRegistry.register_processor>`
381 +to register them to the multi-modal registry:
382 +
383 +```diff
384 + from vllm.model_executor.models.interfaces import SupportsMultiModal
385 ++ from vllm.multimodal import MULTIMODAL_REGISTRY
386 +
387 ++ @MULTIMODAL_REGISTRY.register_processor(YourMultiModalProcessor,
388 ++ info=YourProcessingInfo,
389 ++ dummy_inputs=YourDummyInputsBuilder)
390 + class YourModelForImage2Seq(nn.Module, SupportsMultiModal):
391 ```
docs/source/contributing/model/registration.md
+1 -1
@@ -48,7 +48,7 @@ ModelRegistry.register_model("YourModelForCausalLM", "your_code:YourModelForCaus
48
49 ```{important}
50 If your model is a multimodal model, ensure the model class implements the {class}`~vllm.model_executor.models.interfaces.SupportsMultiModal` interface.
51 -Read more about that [here](#enabling-multimodal-inputs).
51 +Read more about that [here](#supports-multimodal).
52 ```
53
54 ```{note}
docs/source/design/input_processing/input_processing_pipeline.md deleted
-19
@@ -1,19 +0,0 @@
1 -(input-processing-pipeline)=
2 -
3 -# Input Processing Pipeline
4 -
5 -1. Input data is passed to {class}`~vllm.LLMEngine` (or {class}`~vllm.AsyncLLMEngine`).
6 -
7 -2. Tokenize the data if necessary.
8 -
9 -3. Process the inputs using {meth}`INPUT_REGISTRY.process_input <vllm.inputs.registry.InputRegistry.process_input>`.
10 -
11 - - For example, add placeholder tokens to reserve KV cache for multi-modal embeddings.
12 -
13 -4. Send the processed inputs to {class}`~vllm.executor.executor_base.ExecutorBase`.
14 -
15 -5. Distribute the inputs via {class}`~vllm.worker.worker_base.WorkerBase` to {class}`~vllm.worker.model_runner_base.ModelRunnerBase`.
16 -
17 -6. If the data contains multi-modal data, convert it into keyword arguments using {meth}`MULTIMODAL_REGISTRY.map_input <vllm.multimodal.MultiModalRegistry.map_input>`.
18 -
19 - - For example, convert a {class}`PIL.Image.Image` input to its pixel values for a vision model.
docs/source/design/input_processing/model_inputs_index.md deleted
-43
@@ -1,43 +0,0 @@
1 -(input-processing)=
2 -
3 -# Input Processing
4 -
5 -```{eval-rst}
6 -.. currentmodule:: vllm.inputs
7 -```
8 -
9 -Each model can override parts of vLLM's [input processing pipeline](#input-processing-pipeline) via
10 -{data}`~vllm.inputs.INPUT_REGISTRY` and {data}`~vllm.multimodal.MULTIMODAL_REGISTRY`.
11 -
12 -Currently, this mechanism is only utilized in [multi-modal](#multi-modality) models for preprocessing multi-modal input
13 -data in addition to input prompt, but it can be extended to text-only language models when needed.
14 -
15 -## Guides
16 -
17 -```{toctree}
18 -:maxdepth: 1
19 -
20 -input_processing_pipeline
21 -```
22 -
23 -## Module Contents
24 -
25 -### LLM Engine Inputs
26 -
27 -```{eval-rst}
28 -.. autoclass:: vllm.inputs.DecoderOnlyInputs
29 - :members:
30 - :show-inheritance:
31 -```
32 -
33 -### Registry
34 -
35 -```{eval-rst}
36 -.. autodata:: vllm.inputs.INPUT_REGISTRY
37 -```
38 -
39 -```{eval-rst}
40 -.. automodule:: vllm.inputs.registry
41 - :members:
42 - :show-inheritance:
43 -```
docs/source/design/mm_processing.md new
+64
@@ -0,0 +1,64 @@
1 +(mm-processing)=
2 +
3 +# Multi-Modal Data Processing
4 +
5 +To enable various optimizations in vLLM such as [chunked prefill](#chunked-prefill) and [prefix caching](#automatic-prefix-caching), we use {class}`~vllm.multimodal.processing.BaseMultiModalProcessor` to provide the correspondence between placeholder feature tokens (e.g. `<image>`) and multi-modal inputs (e.g. the raw input image) based on the outputs of HF processor.
6 +
7 +Here are the main features of {class}`~vllm.multimodal.processing.BaseMultiModalProcessor`:
8 +
9 +## Prompt Replacement Detection
10 +
11 +One of the main responsibilies of HF processor is to replace input placeholder tokens (e.g. `<image>` for a single image) with feature placeholder tokens (e.g. `<image><image>...<image>`, the number of which equals to the feature size). The information about which tokens have been replaced is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
12 +
13 +In vLLM, this information is specified using {class}`~vllm.multimodal.processing.PromptReplacement` in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`. Given this specification, we can automatically detect whether HF has replaced the input placeholder tokens by checking whether the feature placeholder tokens exist in the prompt.
14 +
15 +## Tokenized Prompt Inputs
16 +
17 +To enable tokenization in a separate process, we support passing input token IDs alongside multi-modal data.
18 +
19 +### The problem
20 +
21 +Consider that HF processors follow these main steps:
22 +
23 +1. Tokenize the text
24 +2. Process multi-modal inputs
25 +3. Perform prompt replacement
26 +
27 +And we require that:
28 +
29 +- For text + multi-modal inputs, apply all steps 1--3.
30 +- For tokenized + multi-modal inputs, apply only steps 2--3.
31 +
32 +How can we achieve this without rewriting HF processors? We can try to call the HF processor several times on different inputs:
33 +
34 +- For text + multi-modal inputs, simply call the HF processor directly.
35 +- For tokenized + multi-modal inputs, call the processor only on the multi-modal inputs.
36 +
37 +While HF processors support text + multi-modal inputs natively, this is not so for tokenized + multi-modal inputs: an error is thrown if the number of input placeholder tokens do not correspond to the number of multi-modal inputs.
38 +
39 +Moreover, since the tokenized text has not passed through the HF processor, we have to apply Step 3 by ourselves to keep the output tokens and multi-modal data consistent with each other.
40 +
41 +(mm-dummy-text)=
42 +
43 +### Dummy text
44 +
45 +We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
46 +
47 +(mm-automatic-prompt-replacement)=
48 +
49 +### Automatic prompt replacement
50 +
51 +We address the second issue by implementing model-agnostic code in
52 +{meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_replacements` to automatically replace input placeholder tokens with feature placeholder tokens based on the specification outputted by {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`.
53 +
54 +### Summary
55 +
56 +With the help of dummy text and automatic prompt replacement, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main`.
57 +
58 +## Processor Output Caching
59 +
60 +Some HF processors, such as the one for Qwen2-VL, are [very slow](gh-issue:9238). To alleviate this problem, we cache the multi-modal outputs of HF processor to avoid processing the same multi-modal input (e.g. image) again.
61 +
62 +When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
63 +
64 +Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#mm-dummy-text) to avoid HF errors. Since this skips HF's prompt replacement code, we apply [automatic prompt replacement](#mm-automatic-prompt-replacement) afterwards to keep the output tokens and multi-modal data consistent with each other.
docs/source/index.md
+1 -1
@@ -145,7 +145,7 @@ design/arch_overview
145 design/huggingface_integration
146 design/plugin_system
147 design/kernel/paged_attention
148 -design/input_processing/model_inputs_index
148 +design/mm_processing
149 design/automatic_prefix_caching
150 design/multiprocessing
151 ```
docs/source/serving/multimodal_inputs.md
+1 -1
@@ -14,7 +14,7 @@ and [open an issue on GitHub](https://github.com/vllm-project/vllm/issues/new/ch
14 To input multi-modal data, follow this schema in {class}`vllm.inputs.PromptType`:
15
16 - `prompt`: The prompt should follow the format that is documented on HuggingFace.
17 -- `multi_modal_data`: This is a dictionary that follows the schema defined in {class}`vllm.multimodal.MultiModalDataDict`.
17 +- `multi_modal_data`: This is a dictionary that follows the schema defined in {class}`vllm.multimodal.inputs.MultiModalDataDict`.
18
19 ### Image
20
vllm/config.py
+1 -2
@@ -2124,8 +2124,7 @@ class MultiModalConfig:
2124
2125 limit_per_prompt: Mapping[str, int] = field(default_factory=dict)
2126 """
2127 - The maximum number of multi-modal input instances allowed per prompt
2128 - for each :class:`~vllm.multimodal.MultiModalPlugin`.
2127 + The maximum number of input items allowed per prompt for each modality.
2128 """
2129
2130 def compute_hash(self) -> str:
vllm/inputs/__init__.py
-3
@@ -11,9 +11,6 @@ INPUT_REGISTRY = InputRegistry()
11 """
12 The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
13 to dispatch data processing according to the target model.
14 -
15 -See also:
16 - :ref:`input-processing-pipeline`
14 """
15
16 __all__ = [
vllm/inputs/registry.py
+2 -10
@@ -313,9 +313,6 @@ class InputRegistry:
313
314 The model is identified by ``model_config``.
315
316 - See also:
317 - :ref:`enabling-multimodal-inputs`
318 -
316 Note:
317 This should be called after
318 :meth:`~MultiModalRegistry.init_mm_limits_per_prompt`.
@@ -384,10 +381,8 @@ class InputRegistry:
381 Register an input processor to a model class.
382
383 The provided function is invoked on each input to the model. This
387 - happens before :meth:`~vllm.multimodal.MultiModalRegistry.map_input`.
388 -
389 - See also:
390 - :ref:`input-processing-pipeline`
384 + happens before
385 + :meth:`~vllm.multimodal.registry.MultiModalRegistry.map_input`.
386 """
387
388 def wrapper(model_cls: N) -> N:
@@ -429,9 +424,6 @@ class InputRegistry:
424 Apply an input processor to an instance of model inputs.
425
426 The model is identified by ``model_config``.
432 -
433 - See also:
434 - :ref:`input-processing-pipeline`
427 """
428 # Avoid circular import
429 from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/__init__.py
+2 -2
@@ -8,10 +8,10 @@ from .registry import MultiModalRegistry
8 MULTIMODAL_REGISTRY = MultiModalRegistry()
9 """
10 The global :class:`~MultiModalRegistry` is used by model runners to
11 -dispatch data processing according to its modality and the target model.
11 +dispatch data processing according to the target model.
12
13 See also:
14 - :ref:`input-processing-pipeline`
14 + :ref:`mm-processing`
15 """
16
17 __all__ = [
vllm/multimodal/base.py
-14
@@ -90,10 +90,6 @@ class MultiModalPlugin(ABC):
90 invoked to transform the data into a dictionary of model inputs.
91
92 If `None` is provided, then the default input mapper is used instead.
93 -
94 - See also:
95 - - :ref:`input-processing-pipeline`
96 - - :ref:`enabling-multimodal-inputs`
93 """
94
95 def wrapper(model_cls: N) -> N:
@@ -126,10 +122,6 @@ class MultiModalPlugin(ABC):
122
123 Raises:
124 TypeError: If the data type is not supported.
129 -
130 - See also:
131 - - :ref:`input-processing-pipeline`
132 - - :ref:`enabling-multimodal-inputs`
125 """
126
127 # Avoid circular import
@@ -186,9 +178,6 @@ class MultiModalPlugin(ABC):
178 for a model class.
179
180 If `None` is provided, then the default calculation is used instead.
189 -
190 - See also:
191 - :ref:`enabling-multimodal-inputs`
181 """
182
183 def wrapper(model_cls: N) -> N:
@@ -218,9 +207,6 @@ class MultiModalPlugin(ABC):
207 If this registry is not applicable to the model, `0` is returned.
208
209 The model is identified by ``model_config``.
221 -
222 - See also:
223 - :ref:`enabling-multimodal-inputs`
210 """
211 # Avoid circular import
212 from vllm.model_executor.model_loader import get_model_architecture
vllm/multimodal/inputs.py
+2 -1
@@ -493,7 +493,8 @@ A dictionary containing placeholder ranges for each modality.
493
494 class MultiModalInputsV2(TypedDict):
495 """
496 - Represents the outputs of :class:`vllm.multimodal.MultiModalProcessor`,
496 + Represents the outputs of
497 + :class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
498 ready to be passed to vLLM internals.
499 """
500
vllm/multimodal/registry.py
+8 -4
@@ -100,8 +100,7 @@ class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
100
101 class MultiModalRegistry:
102 """
103 - A registry that dispatches data processing to the
104 - :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
103 + A registry that dispatches data processing according to the model.
104 """
105
106 DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
@@ -367,8 +366,7 @@ class MultiModalRegistry:
366 invoked to transform the data into a dictionary of model inputs.
367
368 See also:
370 - - :ref:`input-processing-pipeline`
371 - - :ref:`enabling-multimodal-inputs`
369 + :ref:`mm-processing`
370 """
371
372 def wrapper(model_cls: N) -> N:
@@ -398,6 +396,9 @@ class MultiModalRegistry:
396 def has_processor(self, model_config: "ModelConfig") -> bool:
397 """
398 Test whether a multi-modal processor is defined for a specific model.
399 +
400 + See also:
401 + :ref:`mm-processing`
402 """
403 return self._get_model_cls(model_config) in self._processor_factories
404
@@ -408,6 +409,9 @@ class MultiModalRegistry:
409 ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
410 """
411 Create a multi-modal processor for a specific model and tokenizer.
412 +
413 + See also:
414 + :ref:`mm-processing`
415 """
416 model_cls = self._get_model_cls(model_config)
417 factories = self._processor_factories[model_cls]