A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Bugfix] Fix deepseek-vl2 inference with more than 2 images (#13818)
Isotr0py committed
Feb 25, 2025 at 22:03 UTC
6ff518626cd1e873eaa9f10db5650f76b45bb88a
2 files changed
+46
-10
vllm/model_executor/models/deepseek_vl2.py
+42
-8
@@ -25,7 +25,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26
ImageSize, MultiModalDataItems)
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- BaseProcessingInfo, PromptReplacement)
28
+ BaseProcessingInfo, ProcessingCache,
29
+ PromptReplacement)
30
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31
from vllm.sequence import IntermediateTensors
32
from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
@@ -138,18 +139,24 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
139
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
140
return {"image": None}
141
141
- def get_num_image_tokens(self, *, image_width: int,
142
- image_height: int) -> int:
142
+ def get_num_image_tokens(self,
143
+ *,
144
+ image_width: int,
145
+ image_height: int,
146
+ cropping: bool = True) -> int:
147
hf_processor = self.get_hf_processor()
148
image_size = hf_processor.image_size
149
patch_size = hf_processor.patch_size
150
downsample_ratio = hf_processor.downsample_ratio
151
148
- best_width, best_height = hf_processor.select_best_resolution(
149
- (image_width, image_height))
152
+ if cropping:
153
+ best_width, best_height = hf_processor.select_best_resolution(
154
+ (image_width, image_height))
155
+ num_width_tiles, num_height_tiles = (best_width // image_size,
156
+ best_height // image_size)
157
+ else:
158
+ num_width_tiles = num_height_tiles = 1
159
151
- num_width_tiles, num_height_tiles = (best_width // image_size,
152
- best_height // image_size)
160
h = w = math.ceil((image_size // patch_size) / downsample_ratio)
161
162
global_views_tokens = h * (w + 1)
@@ -169,10 +176,12 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
176
seq_len: int,
177
mm_counts: Mapping[str, int],
178
) -> Mapping[str, int]:
179
+ num_images = mm_counts.get("image", 0)
180
max_image_size = self.get_image_size_with_most_features()
181
max_image_tokens = self.get_num_image_tokens(
182
image_height=max_image_size.height,
175
- image_width=max_image_size.width)
183
+ image_width=max_image_size.width,
184
+ cropping=num_images <= 2)
185
186
return {"image": max_image_tokens}
187
@@ -207,6 +216,30 @@ class DeepseekVL2DummyInputsBuilder(
216
class DeepseekVL2MultiModalProcessor(
217
BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]):
218
219
+ def __init__(
220
+ self,
221
+ info: DeepseekVL2ProcessingInfo,
222
+ dummy_inputs: "BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]",
223
+ *,
224
+ cache: Optional[ProcessingCache] = None,
225
+ enable_sanity_checks: bool = True) -> None:
226
+ super().__init__(
227
+ info,
228
+ dummy_inputs,
229
+ cache=cache,
230
+ enable_sanity_checks=enable_sanity_checks,
231
+ )
232
+
233
+ mm_limit = self.info.ctx.model_config.multimodal_config.limit_per_prompt
234
+ if self.cache is not None and mm_limit["image"] > 2:
235
+ # The processor output depends on the number of images passed,
236
+ # making it incompatible with processing cache which is supposed
237
+ # to be invariant of how many images are passed per prompt
238
+ self.cache = None
239
+ logger.warning_once(
240
+ f"{type(self).__name__} does not support processing cache with "
241
+ "image limit larger than 2.")
242
+
243
def _call_hf_processor(
244
self,
245
prompt: str,
@@ -271,6 +304,7 @@ class DeepseekVL2MultiModalProcessor(
304
num_image_tokens = self.info.get_num_image_tokens(
305
image_width=image_size.width,
306
image_height=image_size.height,
307
+ cropping=len(images) <= 2,
308
)
309
return [image_token_id] * num_image_tokens
310
vllm/model_executor/models/h2ovl.py
+4
-2
@@ -477,13 +477,15 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
477
enable_sanity_checks=enable_sanity_checks,
478
)
479
480
- if self.cache is not None:
480
+ mm_limit = self.info.ctx.model_config.multimodal_config.limit_per_prompt
481
+ if self.cache is not None and mm_limit["image"] >= 2:
482
# The processor output depends on the number of images passed,
483
# making it incompatible with processing cache which is supposed
484
# to be invariant of how many images are passed per prompt
485
self.cache = None
486
logger.warning_once(
486
- f"{type(self).__name__} does not support processing cache.")
487
+ f"{type(self).__name__} does not support processing cache with "
488
+ "multi-image support enabled.")
489
490
def _get_prompt_replacements(
491
self,