A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Misc] Fix the size of batched_dummy_mm_inputs in profile_run (#20434)
Signed-off-by: bk-201 <joy25810@foxmail.com>
B-201 committed
Jul 9, 2025 at 11:15 UTC
6bbf1795b73a89a72672785c41a046ac6db9d54f
2 files changed
+9
-6
tests/models/registry.py
+2
-1
@@ -412,7 +412,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
412
hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]}), # noqa: E501
413
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
414
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
415
- "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct"), # noqa: E501
415
+ "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
416
+ max_model_len=4096),
417
"Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
418
"Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
419
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
vllm/v1/worker/gpu_model_runner.py
+7
-5
@@ -2219,8 +2219,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
2219
encoder_budget = min(self.max_num_encoder_input_tokens,
2220
self.encoder_cache_size)
2221
2222
- max_num_mm_items_encoder_budget = cdiv(encoder_budget,
2223
- max_tokens_per_mm_item)
2222
+ max_num_mm_items_encoder_budget = encoder_budget // \
2223
+ max_tokens_per_mm_item
2224
2225
# Check how many items of this modality can be supported by
2226
# the decoder budget.
@@ -2233,8 +2233,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
2233
max_num_mm_items_decoder_budget = self.max_num_reqs * \
2234
max_mm_items_per_req
2235
2236
- max_num_mm_items = min(max_num_mm_items_encoder_budget,
2237
- max_num_mm_items_decoder_budget)
2236
+ max_num_mm_items = max(
2237
+ 1,
2238
+ min(max_num_mm_items_encoder_budget,
2239
+ max_num_mm_items_decoder_budget))
2240
2241
logger.info(
2242
"Encoder cache will be initialized with a budget of %s tokens,"
@@ -2244,7 +2246,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
2246
# Create dummy batch of multimodal inputs.
2247
dummy_mm_kwargs = self.mm_registry.get_decoder_dummy_data(
2248
model_config=self.model_config,
2247
- seq_len=self.max_num_tokens,
2249
+ seq_len=max_tokens_per_mm_item,
2250
mm_counts={
2251
dummy_data_modality: 1
2252
},