@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 83b824c8b4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

[VLM] Remove `BaseProcessingInfo.get_mm_max_tokens_per_item` (#16408)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 11, 2025 at 00:06 UTC 83b824c8b4ee55824b30f0509fd312b0cddb35e5
39 files changed +104 -677
docs/source/contributing/model/multimodal.md
+37 -203
@@ -121,17 +121,21 @@ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
121 return {"image": None, "video": 1}
122 ```
123
124 -### Maximum number of placeholder feature tokens
124 +## 3. Specify dummy inputs
125
126 -Also, override the abstract method {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`
127 -to return the maximum number of placeholder feature tokens per input item for each modality.
126 +Then, inherit {class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` to construct dummy inputs for
127 +HF processing as well as memory profiling.
128
129 -When calling the model, the output embeddings from the visual encoder are assigned to the input positions
130 -containing placeholder feature tokens. Therefore, the number of placeholder feature tokens should be equal
131 -to the size of the output embeddings.
129 +### For memory profiling
130
133 -:::::{tab-set}
134 -::::{tab-item} Basic example: LLaVA
131 +Override the abstract method {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`
132 +to construct dummy inputs for memory profiling. This dummy input should result in the worst-case memory usage of
133 +the model so that vLLM can reserve the correct amount of memory for it.
134 +
135 +Assuming that the memory usage increases with the number of tokens, the dummy input can be constructed to maximize the number of output embeddings, which is the same number as placeholder feature tokens.
136 +
137 +::::{tab-set}
138 +:::{tab-item} Basic example: LLaVA
139 :sync: llava
140
141 Looking at the code of HF's `LlavaForConditionalGeneration`:
@@ -240,7 +244,7 @@ def get_num_image_tokens(
244 ```
245
246 Notice that the number of image tokens doesn't depend on the image width and height.
243 -So, we can calculate the maximum number of image tokens using any image size:
247 +We can simply use a dummy `image_size`:
248
249 ```python
250 def get_image_size_with_most_features(self) -> ImageSize:
@@ -248,33 +252,35 @@ def get_image_size_with_most_features(self) -> ImageSize:
252 width = height = hf_config.image_size
253 return ImageSize(width=width, height=height)
254
251 -def get_max_image_tokens(self) -> int:
252 - target_width, target_height = self.get_image_size_with_most_features()
253 -
254 - return self.get_num_image_tokens(
255 - image_width=target_width,
256 - image_height=target_height,
257 - )
258 -```
259 -
260 -And thus, we can override the method as:
261 -
262 -```python
263 -def get_mm_max_tokens_per_item(
255 +def get_dummy_processor_inputs(
256 self,
257 seq_len: int,
258 mm_counts: Mapping[str, int],
267 -) -> Mapping[str, int]:
268 - return {"image": self.get_max_image_tokens()}
259 +) -> ProcessorInputs:
260 + num_images = mm_counts.get("image", 0)
261 +
262 + processor = self.info.get_hf_processor()
263 + image_token = processor.image_token
264 +
265 + hf_config = self.get_hf_config()
266 + target_width, target_height = self.info.get_image_size_with_most_features()
267 +
268 + mm_data = {
269 + "image":
270 + self._get_dummy_images(width=target_width,
271 + height=target_height,
272 + num_images=num_images)
273 + }
274 +
275 + return ProcessorInputs(
276 + prompt_text=image_token * num_images,
277 + mm_data=mm_data,
278 + )
279 ```
280
271 -:::{note}
272 -Our [actual code](gh-file:vllm/model_executor/models/llava.py) is more abstracted to support vision encoders other than CLIP.
281 :::
282
275 -::::
276 -
277 -::::{tab-item} Non-consecutive feature tokens: Fuyu
283 +:::{tab-item} No input placeholders: Fuyu
284 :sync: fuyu
285
286 Looking at the code of HF's `FuyuForCausalLM`:
@@ -394,188 +400,16 @@ num_patches_per_dim_w = image_width // patch_width
400 num_patches = num_patches_per_dim_h * num_patches_per_dim_w
401 ```
402
397 -We can calculate this in vLLM using this code:
398 -
399 -```python
400 -def get_num_image_patches(
401 - self,
402 - *,
403 - image_width: int,
404 - image_height: int,
405 -) -> int:
406 - image_processor = self.get_image_processor()
407 - target_width = image_processor.size["width"]
408 - target_height = image_processor.size["height"]
409 - patch_width = image_processor.patch_size["width"]
410 - patch_height = image_processor.patch_size["height"]
411 -
412 - if not (image_width <= target_width and image_height <= target_height):
413 - height_scale_factor = target_height / image_height
414 - width_scale_factor = target_width / image_width
415 - optimal_scale_factor = min(height_scale_factor, width_scale_factor)
416 -
417 - image_height = int(image_height * optimal_scale_factor)
418 - image_width = int(image_width * optimal_scale_factor)
419 -
420 - ncols = math.ceil(image_width / patch_width)
421 - nrows = math.ceil(image_height / patch_height)
422 - return ncols * nrows
423 -```
424 -
425 -These image patches correspond to placeholder tokens (`|SPEAKER|`). However, the processor also
426 -inserts newline tokens (`|NEWLINE|`) as shown here:
427 -
428 -```python
429 -# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L654-L670
430 -tensor_of_image_ids = torch.full(
431 - [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device
432 -)
433 -patches = self.patchify_image(image=image.unsqueeze(0)).squeeze(0)
434 -assert num_patches == patches.shape[0]
435 -
436 -if variable_sized:
437 - # Now terminate each line with |NEWLINE|.
438 - tensor_of_image_ids = tensor_of_image_ids.reshape(-1, image_width // patch_width)
439 - newline_ids = torch.full(
440 - [tensor_of_image_ids.shape[0], 1],
441 - image_newline_id,
442 - dtype=torch.int32,
443 - device=image_input.device,
444 - )
445 - tensor_of_image_ids = torch.cat([tensor_of_image_ids, newline_ids], dim=1)
446 - tensor_of_image_ids = tensor_of_image_ids.reshape(-1)
447 -```
448 -
449 -So, the layout of tokens for an image is:
450 -
451 -```
452 -|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
453 -|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
454 -...
455 -|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
456 -```
457 -
458 -This makes the placeholder tokens non-consecutive in the prompt.
459 -Since vLLM requires the feature tokens to be consecutive, **we also treat the newline tokens as feature tokens**.
460 -
461 -So overall, the total number of feature tokens is
462 -
463 -```python
464 -def get_num_image_tokens(
465 - self,
466 - *,
467 - image_width: int,
468 - image_height: int,
469 -) -> int:
470 - image_processor = self.get_image_processor()
471 - target_width = image_processor.size["width"]
472 - target_height = image_processor.size["height"]
473 - patch_width = image_processor.patch_size["width"]
474 - patch_height = image_processor.patch_size["height"]
475 -
476 - if not (image_width <= target_width and image_height <= target_height):
477 - height_scale_factor = target_height / image_height
478 - width_scale_factor = target_width / image_width
479 - optimal_scale_factor = min(height_scale_factor, width_scale_factor)
480 -
481 - image_height = int(image_height * optimal_scale_factor)
482 - image_width = int(image_width * optimal_scale_factor)
483 -
484 - ncols = math.ceil(image_width / patch_width)
485 - nrows = math.ceil(image_height / patch_height)
486 - return (ncols + 1) * nrows
487 -```
488 -
489 -To calculate the maximum number of image tokens, recall that input images are first resized
490 -to fit within `image_processor.size`. The maximum possible dimensions of the image before
491 -being converted into patches is therefore equal to `image_processor.size`.
403 +These image patches correspond to placeholder tokens (`|SPEAKER|`). So, we just need to maximize the number of image patches. Since input images are first resized
404 +to fit within `image_processor.size`, we can maximize the number of image patches by inputting an image with size equal to `image_processor.size`.
405
406 ```python
407 def get_image_size_with_most_features(self) -> ImageSize:
408 image_processor = self.get_image_processor()
409 return ImageSize(width=image_processor.size["width"],
410 height=image_processor.size["height"])
498 -
499 -def get_max_image_tokens(self) -> int:
500 - target_width, target_height = self.get_image_size_with_most_features()
501 -
502 - return self.get_num_image_tokens(
503 - image_width=target_width,
504 - image_height=target_height,
505 - )
506 -```
507 -
508 -And thus, we can override the method as:
509 -
510 -```python
511 -def get_mm_max_tokens_per_item(
512 - self,
513 - seq_len: int,
514 - mm_counts: Mapping[str, int],
515 -) -> Mapping[str, int]:
516 - return {"image": self.get_max_image_tokens()}
517 -```
518 -
519 -:::{note}
520 -Our [actual code](gh-file:vllm/model_executor/models/fuyu.py) returns `ncols` and `nrows` directly instead of the total token count.
521 -This is because `ncols` and `nrows` are used to specify the layout of the feature tokens (as shown in Step 4 of this guide).
522 -:::
523 -
524 -::::
525 -:::::
526 -
527 -## 3. Specify dummy inputs
528 -
529 -Then, inherit {class}`~vllm.multimodal.profiling.BaseDummyInputsBuilder` to construct dummy inputs for
530 -HF processing as well as memory profiling.
531 -
532 -### For memory profiling
533 -
534 -Override the abstract method {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`
535 -to construct dummy inputs for memory profiling. This dummy input should result in the worst-case memory usage of
536 -the model so that vLLM can reserve the correct amount of memory for it.
537 -
538 -Assuming that the memory usage increases with the number of tokens, the dummy input can be constructed based
539 -on the code for {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max_tokens_per_item`.
540 -
541 -::::{tab-set}
542 -:::{tab-item} Basic example: LLaVA
543 -:sync: llava
544 -
545 -Making use of the `get_image_size_with_most_features` method implemented in Step 2:
546 -
547 -```python
548 -def get_dummy_processor_inputs(
549 - self,
550 - seq_len: int,
551 - mm_counts: Mapping[str, int],
552 -) -> ProcessorInputs:
553 - num_images = mm_counts.get("image", 0)
554 -
555 - processor = self.info.get_hf_processor()
556 - image_token = processor.image_token
557 -
558 - hf_config = self.get_hf_config()
559 - target_width, target_height = self.info.get_image_size_with_most_features()
560 -
561 - mm_data = {
562 - "image":
563 - self._get_dummy_images(width=target_width,
564 - height=target_height,
565 - num_images=num_images)
566 - }
567 -
568 - return ProcessorInputs(
569 - prompt_text=image_token * num_images,
570 - mm_data=mm_data,
571 - )
411 ```
412
574 -:::
575 -
576 -:::{tab-item} No input placeholders: Fuyu
577 -:sync: fuyu
578 -
413 Fuyu does not expect image placeholders in the inputs to HF processor, so
414 the dummy prompt text is empty regardless of the number of images.
415 Otherwise, the logic of this method is very similar to LLaVA:
tests/models/multimodal/processing/test_llama4.py
-5
@@ -76,11 +76,6 @@ def test_processor_override(
76 if v == config.boi_token_index]
77
78 # patch sizes and masks
79 - patch_token_id = vocab[hf_processor.img_patch_token]
80 - num_patches = processed_inputs["prompt_token_ids"].count(patch_token_id)
81 - mm_counts = {"image": num_imgs}
82 - assert num_patches / num_imgs <= \
83 - processor.info.get_mm_max_tokens_per_item(32768, mm_counts)["image"]
79 num_patches_per_chunk = processor.info.get_patch_per_chunk(
80 config.vision_config)
81 assert prompt_token_ids.count(config.image_token_index) \
vllm/model_executor/models/aria.py
-7
@@ -408,13 +408,6 @@ class AriaProcessingInfo(BaseProcessingInfo):
408 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
409 return {"image": None}
410
411 - def get_mm_max_tokens_per_item(
412 - self,
413 - seq_len: int,
414 - mm_counts: Mapping[str, int],
415 - ) -> Mapping[str, int]:
416 - return {"image": self.get_num_image_tokens()}
417 -
411 def get_num_image_tokens(self) -> int:
412 hf_config = self.get_hf_config()
413 return max(hf_config.projector_patch_to_query_dict.values())
vllm/model_executor/models/aya_vision.py
-25
@@ -117,31 +117,6 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
117 def get_image_processor(self) -> GotOcr2ImageProcessor:
118 return self.get_hf_processor().image_processor
119
120 - def get_mm_max_tokens_per_item(
121 - self,
122 - seq_len: int,
123 - mm_counts: Mapping[str, int],
124 - ) -> Mapping[str, int]:
125 - return {"image": self.get_max_image_tokens()}
126 -
127 - def get_max_image_tokens(self) -> int:
128 - hf_processor = self.get_hf_processor()
129 - image_processor = hf_processor.image_processor
130 -
131 - image_size = self.get_image_size_with_most_features()
132 - num_patches = self.get_num_patches(
133 - image_width=image_size.width,
134 - image_height=image_size.height,
135 - size=image_processor.size,
136 - min_patches=image_processor.min_patches,
137 - max_patches=image_processor.max_patches,
138 - )
139 -
140 - img_patches_per_tile = (hf_processor.img_size //
141 - hf_processor.patch_size)**2
142 -
143 - return num_patches * img_patches_per_tile
144 -
120 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
121 return {"image": None}
122
vllm/model_executor/models/blip2.py
-7
@@ -406,13 +406,6 @@ class Blip2ProcessingInfo(BaseProcessingInfo):
406 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
407 return {"image": 1}
408
409 - def get_mm_max_tokens_per_item(
410 - self,
411 - seq_len: int,
412 - mm_counts: Mapping[str, int],
413 - ) -> Mapping[str, int]:
414 - return {"image": self.get_num_image_tokens()}
415 -
409 def get_num_image_tokens(self) -> int:
410 hf_config = self.get_hf_config()
411 return hf_config.num_query_tokens
vllm/model_executor/models/chameleon.py
-7
@@ -64,13 +64,6 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
64 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
65 return {"image": 1}
66
67 - def get_mm_max_tokens_per_item(
68 - self,
69 - seq_len: int,
70 - mm_counts: Mapping[str, int],
71 - ) -> Mapping[str, int]:
72 - return {"image": self.get_num_image_tokens()}
73 -
67 def get_num_image_tokens(self) -> int:
68 processor = self.get_hf_processor()
69 return processor.image_seq_length
vllm/model_executor/models/clip.py
-3
@@ -30,9 +30,6 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
30 ) -> int:
31 return self.get_patch_grid_length()**2 + 1
32
33 - def get_max_image_tokens(self) -> int:
34 - return self.get_patch_grid_length()**2 + 1
35 -
33 def get_image_size(self) -> int:
34 return self.vision_config.image_size
35
vllm/model_executor/models/deepseek_vl2.py
-14
@@ -168,20 +168,6 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
168 image_width=x[1], image_height=x[0]))
169 return ImageSize(width=width, height=height)
170
171 - def get_mm_max_tokens_per_item(
172 - self,
173 - seq_len: int,
174 - mm_counts: Mapping[str, int],
175 - ) -> Mapping[str, int]:
176 - num_images = mm_counts.get("image", 0)
177 - max_image_size = self.get_image_size_with_most_features()
178 - max_image_tokens = self.get_num_image_tokens(
179 - image_height=max_image_size.height,
180 - image_width=max_image_size.width,
181 - cropping=num_images <= 2)
182 -
183 - return {"image": max_image_tokens}
184 -
171
172 class DeepseekVL2DummyInputsBuilder(
173 BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]):
vllm/model_executor/models/florence2.py
+2 -9
@@ -764,17 +764,10 @@ class Florence2ProcessingInfo(BaseProcessingInfo):
764 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
765 return {"image": 1}
766
767 - def get_max_image_tokens(self) -> int:
767 + def get_num_image_tokens(self) -> int:
768 processor_config = self.ctx.get_hf_image_processor_config()
769 return processor_config["image_seq_length"]
770
771 - def get_mm_max_tokens_per_item(
772 - self,
773 - seq_len: int,
774 - mm_counts: Mapping[str, int],
775 - ) -> Mapping[str, int]:
776 - return {"image": self.get_max_image_tokens()}
777 -
771
772 class Florence2DummyInputsBuilder(
773 BaseDummyInputsBuilder[Florence2ProcessingInfo]):
@@ -871,7 +864,7 @@ class Florence2MultiModalProcessor(
864 ) -> Sequence[PromptUpdate]:
865 hf_config = self.info.get_hf_config()
866 pad_token_id = hf_config.pad_token_id
874 - num_image_tokens = self.info.get_max_image_tokens()
867 + num_image_tokens = self.info.get_num_image_tokens()
868 image_tokens = [pad_token_id] * num_image_tokens
869
870 return [
vllm/model_executor/models/fuyu.py
-15
@@ -80,13 +80,6 @@ class FuyuProcessingInfo(BaseProcessingInfo):
80 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
81 return {"image": 1}
82
83 - def get_mm_max_tokens_per_item(
84 - self,
85 - seq_len: int,
86 - mm_counts: Mapping[str, int],
87 - ) -> Mapping[str, int]:
88 - return {"image": self.get_max_image_tokens()}
89 -
83 def get_image_feature_grid_size(
84 self,
85 *,
@@ -129,14 +122,6 @@ class FuyuProcessingInfo(BaseProcessingInfo):
122 return ImageSize(width=image_processor.size["width"],
123 height=image_processor.size["height"])
124
132 - def get_max_image_tokens(self) -> int:
133 - target_width, target_height = self.get_image_size_with_most_features()
134 -
135 - return self.get_num_image_tokens(
136 - image_width=target_width,
137 - image_height=target_height,
138 - )
139 -
125
126 class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
127
vllm/model_executor/models/gemma3_mm.py
-16
@@ -68,13 +68,6 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
68 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
69 return {"image": None}
70
71 - def get_mm_max_tokens_per_item(
72 - self,
73 - seq_len: int,
74 - mm_counts: Mapping[str, int],
75 - ) -> Mapping[str, int]:
76 - return {"image": self.get_max_image_tokens()}
77 -
71 def _resolve_image_kwargs(
72 self,
73 processor: Gemma3Processor,
@@ -228,15 +221,6 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
221 # Result in the max possible feature size (h:w = max_num_crops:1)
222 return ImageSize(height=50 * max_num_crops, width=50)
223
231 - def get_max_image_tokens(self) -> int:
232 - target_width, target_height = self.get_image_size_with_most_features()
233 -
234 - return self.get_num_image_tokens(
235 - image_width=target_width,
236 - image_height=target_height,
237 - processor=None,
238 - )
239 -
224
225 class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
226
vllm/model_executor/models/glm4v.py
-7
@@ -431,13 +431,6 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
431 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
432 return {"image": 1}
433
434 - def get_mm_max_tokens_per_item(
435 - self,
436 - seq_len: int,
437 - mm_counts: Mapping[str, int],
438 - ) -> Mapping[str, int]:
439 - return {"image": self.get_num_image_feature_tokens()}
440 -
434 def get_num_image_tokens(self) -> int:
435 hf_config = self.get_hf_config()
436 vision_config = hf_config.vision_config
vllm/model_executor/models/h2ovl.py
-23
@@ -412,19 +412,6 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
412 **kwargs,
413 )
414
415 - def get_mm_max_tokens_per_item(
416 - self,
417 - seq_len: int,
418 - mm_counts: Mapping[str, int],
419 - ) -> Mapping[str, int]:
420 - max_tokens_one_image = self.get_max_image_tokens(use_msac=None)
421 - if mm_counts.get("image", 0) <= 1:
422 - max_tokens_per_image = max_tokens_one_image
423 - else:
424 - max_tokens_per_image = self.get_max_image_tokens(use_msac=False)
425 -
426 - return {"image": max_tokens_per_image}
427 -
415 def get_num_image_tokens(
416 self,
417 *,
@@ -442,16 +429,6 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
429 use_msac=use_msac,
430 )
431
445 - def get_max_image_tokens(self, use_msac: Optional[bool] = None) -> int:
446 - target_width, target_height = self.get_image_size_with_most_features()
447 -
448 - return self.get_num_image_tokens(
449 - image_width=target_width,
450 - image_height=target_height,
451 - processor=None,
452 - use_msac=use_msac,
453 - )
454 -
432
433 class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
434 ):
vllm/model_executor/models/idefics3.py
-16
@@ -97,13 +97,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
97 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
98 return {"image": None}
99
100 - def get_mm_max_tokens_per_item(
101 - self,
102 - seq_len: int,
103 - mm_counts: Mapping[str, int],
104 - ) -> Mapping[str, int]:
105 - return {"image": self.get_max_image_tokens()}
106 -
100 def _resize_output_size(self,
101 *,
102 height: int,
@@ -287,15 +280,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
280 height=image_processor.size["longest_edge"],
281 )
282
290 - def get_max_image_tokens(self) -> int:
291 - target_width, target_height = self.get_image_size_with_most_features()
292 -
293 - return self.get_num_image_tokens(
294 - image_width=target_width,
295 - image_height=target_height,
296 - processor=None,
297 - )
298 -
283
284 class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo]
285 ):
vllm/model_executor/models/internvl.py
-16
@@ -458,13 +458,6 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
458 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
459 return {"image": None}
460
461 - def get_mm_max_tokens_per_item(
462 - self,
463 - seq_len: int,
464 - mm_counts: Mapping[str, int],
465 - ) -> Mapping[str, int]:
466 - return {"image": self.get_max_image_tokens()}
467 -
461 def get_num_image_tokens(
462 self,
463 *,
@@ -480,15 +473,6 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
473 image_height=image_height,
474 )
475
483 - def get_max_image_tokens(self) -> int:
484 - target_width, target_height = self.get_image_size_with_most_features()
485 -
486 - return self.get_num_image_tokens(
487 - image_width=target_width,
488 - image_height=target_height,
489 - processor=None,
490 - )
491 -
476 def get_image_size_with_most_features(self) -> ImageSize:
477 processor = self.get_hf_processor()
478
vllm/model_executor/models/llava.py
-7
@@ -137,13 +137,6 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
137 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
138 return {"image": None}
139
140 - def get_mm_max_tokens_per_item(
141 - self,
142 - seq_len: int,
143 - mm_counts: Mapping[str, int],
144 - ) -> Mapping[str, int]:
145 - return {"image": self.get_max_image_tokens()}
146 -
140 def _apply_feature_select_strategy(
141 self,
142 strategy: str,
vllm/model_executor/models/llava_next_video.py
-16
@@ -61,22 +61,6 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
61 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
62 return {"video": 1}
63
64 - def get_mm_max_tokens_per_item(
65 - self,
66 - seq_len: int,
67 - mm_counts: Mapping[str, int],
68 - ) -> Mapping[str, int]:
69 - target_width, target_height = self.get_image_size_with_most_features()
70 -
71 - max_video_tokens = self.get_num_video_tokens(
72 - image_width=target_width,
73 - image_height=target_height,
74 - num_frames=self.get_num_frames_with_most_features(
75 - seq_len, mm_counts),
76 - )
77 -
78 - return {"video": max_video_tokens}
79 -
64 def get_image_size_with_most_features(self) -> ImageSize:
65 vision_encoder_info = self.get_vision_encoder_info()
66 width = height = vision_encoder_info.get_image_size()
vllm/model_executor/models/llava_onevision.py
-10
@@ -101,16 +101,6 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
101 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
102 return {"image": None, "video": None}
103
104 - def get_mm_max_tokens_per_item(
105 - self,
106 - seq_len: int,
107 - mm_counts: Mapping[str, int],
108 - ) -> Mapping[str, int]:
109 - return {
110 - "image": self.get_max_image_tokens(),
111 - "video": self.get_max_video_tokens(seq_len, mm_counts),
112 - }
113 -
104 # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
105 # with additional logic afterwards taken from LlavaOnevisionProcessor
106 def _get_num_unpadded_features(
vllm/model_executor/models/minicpmo.py
-11
@@ -142,17 +142,6 @@ class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
142 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
143 return {**super().get_supported_mm_limits(), "audio": None}
144
145 - def get_mm_max_tokens_per_item(
146 - self,
147 - seq_len: int,
148 - mm_counts: Mapping[str, int],
149 - ) -> Mapping[str, int]:
150 - return {
151 - **super().get_mm_max_tokens_per_item(seq_len, mm_counts),
152 - "audio":
153 - self.get_max_audio_tokens(),
154 - }
155 -
145 def get_audio_placeholder(
146 self,
147 audio_lens: int,
vllm/model_executor/models/minicpmv.py
-12
@@ -346,18 +346,6 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
346
347 return mm_limits
348
349 - def get_mm_max_tokens_per_item(
350 - self,
351 - seq_len: int,
352 - mm_counts: Mapping[str, int],
353 - ) -> Mapping[str, int]:
354 - mm_max_tokens = {"image": self.get_max_image_tokens()}
355 - if self.get_model_version() == (2, 6):
356 - mm_max_tokens["video"] = self.get_max_video_tokens(
357 - seq_len, mm_counts)
358 -
359 - return mm_max_tokens
360 -
349 def get_slice_image_placeholder(
350 self,
351 image_size: ImageSize,
vllm/model_executor/models/mistral3.py
-15
@@ -162,13 +162,6 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
162 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
163 return {"image": None}
164
165 - def get_mm_max_tokens_per_item(
166 - self,
167 - seq_len: int,
168 - mm_counts: Mapping[str, int],
169 - ) -> Mapping[str, int]:
170 - return {"image": self.get_max_image_tokens()}
171 -
165 def get_num_image_tokens(
166 self,
167 *,
@@ -186,14 +179,6 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
179 width = height = vision_encoder_info.get_image_size()
180 return ImageSize(width=width, height=height)
181
189 - def get_max_image_tokens(self) -> int:
190 - target_width, target_height = self.get_image_size_with_most_features()
191 -
192 - return self.get_num_image_tokens(
193 - image_width=target_width,
194 - image_height=target_height,
195 - )
196 -
182
183 _I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
184
vllm/model_executor/models/mllama.py
-10
@@ -106,16 +106,6 @@ class MllamaProcessingInfo(BaseProcessingInfo):
106 image_size = self.get_hf_config().vision_config.image_size
107 return calc_token_per_chunk(image_size)
108
109 - def get_mm_max_tokens_per_item(
110 - self,
111 - seq_len: int,
112 - mm_counts: Mapping[str, int],
113 - ) -> Mapping[str, int]:
114 - vision_config = self.get_hf_config().vision_config
115 - token_per_chunk = self.get_token_per_chunk_from_config()
116 - mm_max_tokens = vision_config.max_num_tiles * token_per_chunk
117 - return {"image": mm_max_tokens}
118 -
109 def get_num_tiles_per_image(self, image_height: int,
110 image_width: int) -> int:
111 vision_config = self.get_hf_config().vision_config
vllm/model_executor/models/mllama4.py
-19
@@ -498,17 +498,6 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
498 image_processor = self.get_hf_processor().image_processor
499 return image_processor.max_patches
500
501 - def get_mm_max_tokens_per_item(
502 - self,
503 - seq_len: int,
504 - mm_counts: Mapping[str, int],
505 - ) -> Mapping[str, int]:
506 - vision_config = self.get_hf_config().vision_config
507 - patch_per_chunk = self.get_patch_per_chunk(vision_config)
508 - num_patches = self.get_max_num_tiles() + 1
509 -
510 - return {"image": patch_per_chunk * num_patches}
511 -
501 def get_image_size_with_most_features(self) -> ImageSize:
502 vision_config = self.get_hf_config().vision_config
503 image_size = vision_config.image_size
@@ -516,14 +505,6 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
505 return ImageSize(height=self.get_max_num_tiles() * image_size,
506 width=image_size)
507
519 - def get_max_image_tokens(self) -> int:
520 - target_width, target_height = self.get_image_size_with_most_features()
521 -
522 - return self.get_num_image_tokens(
523 - image_width=target_width,
524 - image_height=target_height,
525 - )
526 -
508
509 class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo]
510 ):
vllm/model_executor/models/molmo.py
-16
@@ -1164,13 +1164,6 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1164 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
1165 return {"image": None}
1166
1167 - def get_mm_max_tokens_per_item(
1168 - self,
1169 - seq_len: int,
1170 - mm_counts: Mapping[str, int],
1171 - ) -> Mapping[str, int]:
1172 - return {"image": self.get_max_image_tokens()}
1173 -
1167 def get_num_image_tokens(
1168 self,
1169 *,
@@ -1195,15 +1188,6 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1188
1189 return extra + joint
1190
1198 - def get_max_image_tokens(self) -> int:
1199 - target_width, target_height = self.get_image_size_with_most_features()
1200 -
1201 - return self.get_num_image_tokens(
1202 - image_width=target_width,
1203 - image_height=target_height,
1204 - processor=None,
1205 - )
1206 -
1191 def get_image_size_with_most_features(self) -> ImageSize:
1192 processor = self.get_hf_processor()
1193
vllm/model_executor/models/paligemma.py
+33 -15
@@ -13,7 +13,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
13 from vllm.multimodal import MULTIMODAL_REGISTRY
14 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
15 MultiModalInputs, MultiModalKwargs)
16 -from vllm.multimodal.parse import MultiModalDataItems
16 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
17 + MultiModalDataItems)
18 from vllm.multimodal.processing import (BaseMultiModalProcessor,
19 BaseProcessingInfo, PromptIndexTargets,
20 PromptInsertion, PromptUpdate,
@@ -72,16 +73,18 @@ class PaliGemmaProcessingInfo(BaseProcessingInfo):
73 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
74 return {"image": 1}
75
75 - def get_mm_max_tokens_per_item(
76 + def get_num_image_tokens(
77 self,
77 - seq_len: int,
78 - mm_counts: Mapping[str, int],
79 - ) -> Mapping[str, int]:
80 - return {"image": self.get_num_image_tokens()}
81 -
82 - def get_num_image_tokens(self) -> int:
78 + *,
79 + image_width: int,
80 + image_height: int,
81 + ) -> int:
82 vision_encoder_info = self.get_vision_encoder_info()
84 - return vision_encoder_info.get_max_image_tokens()
83 +
84 + return vision_encoder_info.get_num_image_tokens(
85 + image_width=image_width,
86 + image_height=image_height,
87 + )
88
89
90 class PaliGemmaDummyInputsBuilder(
@@ -148,12 +151,30 @@ class PaliGemmaMultiModalProcessor(
151 image_token_id = hf_config.image_token_index
152
153 tokenizer = self.info.get_tokenizer()
151 - num_image_tokens = self.info.get_num_image_tokens()
152 - image_tokens = [image_token_id] * num_image_tokens
154
155 bos_token_id = tokenizer.bos_token_id
156 assert isinstance(bos_token_id, int)
157
158 + def get_insertion(item_idx: int):
159 + images = mm_items.get_items(
160 + "image", (ImageEmbeddingItems, ImageProcessorItems))
161 +
162 + if isinstance(images, ImageEmbeddingItems):
163 + num_image_tokens = images.get_feature_size(item_idx)
164 + else:
165 + image_size = images.get_image_size(item_idx)
166 + num_image_tokens = self.info.get_num_image_tokens(
167 + image_width=image_size.width,
168 + image_height=image_size.height,
169 + )
170 +
171 + image_tokens = [image_token_id] * num_image_tokens
172 +
173 + return PromptUpdateDetails.select_token_id(
174 + image_tokens + [bos_token_id],
175 + embed_token_id=image_token_id,
176 + )
177 +
178 # Paligemma 1 and 2 have different tokenizer.add_bos_token
179 # Insert <image>*n + <bos> after <bos> for Paligemma 1
180 # Insert <image>*n + <bos> for Paligemma 2
@@ -162,10 +183,7 @@ class PaliGemmaMultiModalProcessor(
183 modality="image",
184 target=PromptIndexTargets.prefix(
185 [bos_token_id] if tokenizer.add_bos_token else []),
165 - insertion=PromptUpdateDetails.select_token_id(
166 - image_tokens + [bos_token_id],
167 - embed_token_id=image_token_id,
168 - ),
186 + insertion=get_insertion,
187 )
188 ]
189
vllm/model_executor/models/phi3v.py
-15
@@ -321,21 +321,6 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
321 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
322 return {"image": None}
323
324 - def get_mm_max_tokens_per_item(
325 - self,
326 - seq_len: int,
327 - mm_counts: Mapping[str, int],
328 - ) -> Mapping[str, int]:
329 - target_width, target_height = self.get_image_size_with_most_features()
330 -
331 - max_image_tokens = self.get_num_image_tokens(
332 - image_width=target_width,
333 - image_height=target_height,
334 - processor=None,
335 - )
336 -
337 - return {"image": max_image_tokens}
338 -
324 def get_num_image_tokens(
325 self,
326 *,
vllm/model_executor/models/pixtral.py
-23
@@ -167,13 +167,6 @@ class PixtralProcessingInfo(BaseProcessingInfo):
167 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
168 return {"image": None}
169
170 - def get_mm_max_tokens_per_item(
171 - self,
172 - seq_len: int,
173 - mm_counts: Mapping[str, int],
174 - ) -> Mapping[str, int]:
175 - return {"image": self.get_max_image_tokens()}
176 -
170 def get_vision_config(
171 self,
172 processor: Optional[PixtralProcessorAdapter] = None,
@@ -207,14 +200,6 @@ class PixtralProcessingInfo(BaseProcessingInfo):
200
201 return ImageSize(width=max_image_size, height=max_image_size)
202
210 - def get_max_image_tokens(self) -> int:
211 - target_width, target_height = self.get_image_size_with_most_features()
212 -
213 - return self.get_num_image_tokens(
214 - image_width=target_width,
215 - image_height=target_height,
216 - )
217 -
203
204 class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
205
@@ -938,14 +923,6 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
923 )
924 return ncols * nrows
925
941 - def get_max_image_tokens(self) -> int:
942 - image_size = self.get_image_size()
943 -
944 - return self.get_num_image_tokens(
945 - image_width=image_size,
946 - image_height=image_size,
947 - )
948 -
926 def get_image_size(self) -> int:
927 return self.vision_config.image_size
928
vllm/model_executor/models/prithvi_geospatial_mae.py
-3
@@ -45,9 +45,6 @@ class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
45 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
46 return {"image": None}
47
48 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
49 - return {"image": 0}
50 -
48
49 class PrithviGeoSpatialMAEInputBuilder(
50 BaseDummyInputsBuilder[PrithviGeoSpatialMAEProcessingInfo]):
vllm/model_executor/models/qwen2_audio.py
-11
@@ -109,17 +109,6 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
109 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
110 return {"audio": None}
111
112 - def get_mm_max_tokens_per_item(
113 - self,
114 - seq_len: int,
115 - mm_counts: Mapping[str, int],
116 - ) -> Mapping[str, int]:
117 - hf_config = self.get_hf_config()
118 - max_source_positions = hf_config.audio_config.max_source_positions
119 - max_output_lengths = (max_source_positions - 2) // 2 + 1
120 -
121 - return {"audio": max_output_lengths}
122 -
112
113 class Qwen2AudioDummyInputsBuilder(
114 BaseDummyInputsBuilder[Qwen2AudioProcessingInfo]):
vllm/model_executor/models/qwen2_vl.py
-10
@@ -818,16 +818,6 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
818 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
819 return {"image": None, "video": None}
820
821 - def get_mm_max_tokens_per_item(
822 - self,
823 - seq_len: int,
824 - mm_counts: Mapping[str, int],
825 - ) -> Mapping[str, int]:
826 - return {
827 - "image": self.get_max_image_tokens(),
828 - "video": self.get_max_video_tokens(seq_len, mm_counts),
829 - }
830 -
821 def _get_vision_info(
822 self,
823 *,
vllm/model_executor/models/qwen_vl.py
-7
@@ -530,13 +530,6 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
530 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
531 return {"image": None}
532
533 - def get_mm_max_tokens_per_item(
534 - self,
535 - seq_len: int,
536 - mm_counts: Mapping[str, int],
537 - ) -> Mapping[str, int]:
538 - return {"image": self.get_num_image_tokens()}
539 -
533 def get_num_image_tokens(self) -> int:
534 hf_config = self.get_hf_config()
535 vision_config = hf_config.visual
vllm/model_executor/models/siglip.py
-3
@@ -33,9 +33,6 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
33 ) -> int:
34 return self.get_patch_grid_length()**2
35
36 - def get_max_image_tokens(self) -> int:
37 - return self.get_patch_grid_length()**2
38 -
36 def get_image_size(self) -> int:
37 return self.vision_config.image_size
38
vllm/model_executor/models/skyworkr1v.py
-16
@@ -459,13 +459,6 @@ class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
459 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
460 return {"image": None}
461
462 - def get_mm_max_tokens_per_item(
463 - self,
464 - seq_len: int,
465 - mm_counts: Mapping[str, int],
466 - ) -> Mapping[str, int]:
467 - return {"image": self.get_max_image_tokens()}
468 -
462 def get_num_image_tokens(
463 self,
464 *,
@@ -481,15 +474,6 @@ class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
474 image_height=image_height,
475 )
476
484 - def get_max_image_tokens(self) -> int:
485 - target_width, target_height = self.get_image_size_with_most_features()
486 -
487 - return self.get_num_image_tokens(
488 - image_width=target_width,
489 - image_height=target_height,
490 - processor=None,
491 - )
492 -
477 def get_image_size_with_most_features(self) -> ImageSize:
478 processor = self.get_hf_processor()
479
vllm/model_executor/models/ultravox.py
-12
@@ -2,7 +2,6 @@
2
3 # Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
4 """PyTorch Ultravox model."""
5 -import math
5 from collections.abc import Iterable, Mapping, Sequence
6 from functools import cached_property
7 from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union
@@ -107,17 +106,6 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
106 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
107 return {"audio": None}
108
110 - def get_mm_max_tokens_per_item(
111 - self,
112 - seq_len: int,
113 - mm_counts: Mapping[str, int],
114 - ) -> Mapping[str, int]:
115 - feature_extractor = self.get_feature_extractor()
116 - max_audio_tokens = math.ceil(feature_extractor.chunk_length *
117 - _AUDIO_TOKENS_PER_SECOND)
118 -
119 - return {"audio": max_audio_tokens * _MAX_ENCODER_BATCH_SIZE}
120 -
109
110 class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
111 ):
vllm/model_executor/models/vision.py
-4
@@ -33,10 +33,6 @@ class VisionEncoderInfo(ABC, Generic[_C]):
33 ) -> int:
34 raise NotImplementedError
35
36 - @abstractmethod
37 - def get_max_image_tokens(self) -> int:
38 - raise NotImplementedError
39 -
36 @abstractmethod
37 def get_image_size(self) -> int:
38 raise NotImplementedError
vllm/model_executor/models/whisper.py
+2 -9
@@ -538,16 +538,9 @@ class WhisperProcessingInfo(BaseProcessingInfo):
538 assert isinstance(feature_extractor, WhisperFeatureExtractor)
539 return feature_extractor
540
541 - def get_max_audio_tokens(self) -> int:
541 + def get_num_audio_tokens(self) -> int:
542 return self.get_hf_config().max_source_positions
543
544 - def get_mm_max_tokens_per_item(
545 - self,
546 - seq_len: int,
547 - mm_counts: Mapping[str, int],
548 - ) -> Mapping[str, int]:
549 - return {"audio": self.get_max_audio_tokens()}
550 -
544
545 class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
546
@@ -630,7 +623,7 @@ class WhisperMultiModalProcessor(
623 hf_processor_mm_kwargs: Mapping[str, object],
624 out_mm_kwargs: MultiModalKwargs,
625 ) -> Sequence[PromptUpdate]:
633 - num_tokens = self.info.get_max_audio_tokens()
626 + num_tokens = self.info.get_num_audio_tokens()
627 return [
628 PromptReplacement(
629 modality="audio",
vllm/multimodal/processing.py
-15
@@ -1034,21 +1034,6 @@ class BaseProcessingInfo:
1034 """
1035 raise NotImplementedError
1036
1037 - @abstractmethod
1038 - def get_mm_max_tokens_per_item(
1039 - self,
1040 - seq_len: int,
1041 - mm_counts: Mapping[str, int],
1042 - ) -> Mapping[str, int]:
1043 - """
1044 - Get the maximum possible number of tokens per data item
1045 - for each modality.
1046 -
1047 - The dictionary returned by this method should have the same
1048 - keys as that returned by :meth:`get_supported_mm_limits`.
1049 - """
1050 - raise NotImplementedError
1051 -
1037
1038 _I = TypeVar("_I", bound=BaseProcessingInfo)
1039
vllm/multimodal/profiling.py
+22 -43
@@ -68,7 +68,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
68 ) -> ProcessorInputs:
69 """
70 Build the input which, after processing, results in
71 - :code:`self.info.get_mm_max_tokens_per_item()` placeholder tokens.
71 + the maximum possible number of placeholder tokens.
72 """
73 raise NotImplementedError
74
@@ -152,8 +152,11 @@ class MultiModalProfiler(Generic[_I]):
152 def _get_dummy_mm_inputs(
153 self,
154 seq_len: int,
155 - mm_counts: Mapping[str, int],
155 + mm_counts: Optional[Mapping[str, int]] = None,
156 ) -> MultiModalInputs:
157 + if mm_counts is None:
158 + mm_counts = self.get_mm_limits()
159 +
160 factory = self.dummy_inputs
161 processor_inputs = factory.get_dummy_processor_inputs(
162 seq_len, mm_counts)
@@ -164,53 +167,23 @@ class MultiModalProfiler(Generic[_I]):
167 hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
168 )
169
167 - def get_and_validate_mm_inputs(
170 + def _get_mm_num_tokens(
171 self,
169 - seq_len: int,
170 - mm_counts: Optional[Mapping[str, int]] = None,
171 - ) -> tuple[MultiModalInputs, Mapping[str, int]]:
172 - if mm_counts is None:
173 - mm_counts = self.get_mm_limits()
174 -
175 - info = self.processing_info
176 - mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(
177 - seq_len, mm_counts)
178 -
179 - if mm_counts.keys() - mm_max_tokens_per_item.keys():
180 - raise AssertionError(
181 - "The keys returned by `get_supported_mm_limits` "
182 - f"({set(mm_counts.keys())}) should be a subset of those "
183 - "returned by `get_mm_max_tokens_per_item` "
184 - f"({set(mm_max_tokens_per_item.keys())})")
185 -
186 - mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
172 + mm_inputs: MultiModalInputs,
173 + ) -> Mapping[str, int]:
174 placeholders_by_modality = mm_inputs["mm_placeholders"]
175
189 - total_placeholders_by_modality = {
176 + return {
177 modality: sum(item.get_num_embeds() for item in placeholders)
178 for modality, placeholders in placeholders_by_modality.items()
179 }
193 - expected_placeholders_by_modality = {
194 - modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
195 - for modality in placeholders_by_modality
196 - }
197 - if total_placeholders_by_modality != expected_placeholders_by_modality:
198 - raise AssertionError(
199 - f"The processed dummy data has a total of "
200 - f"{total_placeholders_by_modality} placeholder tokens, which "
201 - f"is not the expected {expected_placeholders_by_modality} "
202 - "tokens.")
203 - return mm_inputs, total_placeholders_by_modality
180
181 def get_encoder_dummy_data(
182 self,
183 seq_len: int,
184 mm_counts: Optional[Mapping[str, int]] = None,
185 ) -> DummyEncoderData:
210 - (
211 - mm_inputs,
212 - total_placeholders_by_modality,
213 - ) = self.get_and_validate_mm_inputs(seq_len, mm_counts)
186 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
187 mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)
188
189 # For encoder-decoder models, use encoder prompt token ids instead of
@@ -232,7 +205,7 @@ class MultiModalProfiler(Generic[_I]):
205 " is too short "
206 "to hold the multi-modal embeddings in the worst case "
207 f"({total_len} tokens in total, out of which "
235 - f"{total_placeholders_by_modality} are reserved for "
208 + f"{self._get_mm_num_tokens(mm_inputs)} are reserved for "
209 "multi-modal embeddings). This may cause certain "
210 "multi-modal inputs to fail during inference, even when "
211 "the input text is short. To avoid this, you should "
@@ -246,10 +219,7 @@ class MultiModalProfiler(Generic[_I]):
219 seq_len: int,
220 mm_counts: Optional[Mapping[str, int]] = None,
221 ) -> DummyDecoderData:
249 - (
250 - mm_inputs,
251 - total_placeholders_by_modality,
252 - ) = self.get_and_validate_mm_inputs(seq_len, mm_counts)
222 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
223
224 prompt_token_ids = mm_inputs["prompt_token_ids"]
225 total_len = len(prompt_token_ids)
@@ -263,7 +233,7 @@ class MultiModalProfiler(Generic[_I]):
233 "is too short "
234 "to hold the multi-modal embeddings in the worst case "
235 f"({total_len} tokens in total, out of which "
266 - f"{total_placeholders_by_modality} are reserved for "
236 + f"{self._get_mm_num_tokens(mm_inputs)} are reserved for "
237 "multi-modal embeddings). This may cause certain "
238 "multi-modal inputs to fail during inference, even when "
239 "the input text is short. To avoid this, you should "
@@ -278,3 +248,12 @@ class MultiModalProfiler(Generic[_I]):
248 multi_modal_data=mm_inputs["mm_kwargs"],
249 multi_modal_placeholders=mm_inputs["mm_placeholders"],
250 )
251 +
252 + def get_mm_max_tokens(
253 + self,
254 + seq_len: int,
255 + mm_counts: Optional[Mapping[str, int]] = None,
256 + ) -> Mapping[str, int]:
257 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
258 +
259 + return self._get_mm_num_tokens(mm_inputs)
vllm/multimodal/registry.py
+8 -2
@@ -258,10 +258,16 @@ class MultiModalRegistry:
258 """
259 if self.has_processor(model_config):
260 processor = self.create_processor(model_config, disable_cache=True)
261 + profiler = MultiModalProfiler(processor)
262 +
263 seq_len = model_config.max_model_len
264 mm_limits = self.get_mm_limits_per_prompt(model_config)
263 - return processor.info.get_mm_max_tokens_per_item(
264 - seq_len, mm_limits)
265 +
266 + return profiler.get_mm_max_tokens(
267 + seq_len,
268 + {modality: 1
269 + for modality in mm_limits},
270 + )
271
272 return {
273 key: plugin.get_max_multimodal_tokens(model_config)