A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Enforce valid max_num_batched_tokens when disable_chunked_mm_input=True (#16447)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Apr 11, 2025 at 02:09 UTC
aa3b3d76e0db63a4214b45805dc9bc3e5609c30e
3 files changed
+18
-1
tests/v1/core/test_scheduler.py
+9
@@ -322,6 +322,15 @@ def test_no_mm_input_chunking():
322
assert len(output.finished_req_ids) == 0
323
assert output.num_scheduled_tokens[requests[0].request_id] == 800
324
325
+ # Test that we fail if we disable chunked mm input and use too small
326
+ # of a max_num_batched_tokens for the mm input.
327
+ with pytest.raises(ValueError):
328
+ _ = create_scheduler(
329
+ model="llava-hf/llava-1.5-7b-hf",
330
+ max_num_batched_tokens=100,
331
+ disable_chunked_mm_input=True,
332
+ )
333
+
334
335
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
336
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
vllm/engine/arg_utils.py
+1
-1
@@ -1030,7 +1030,7 @@ class EngineArgs:
1030
action=StoreBoolean,
1031
default=EngineArgs.disable_chunked_mm_input,
1032
nargs="?",
1033
- const="False",
1033
+ const="True",
1034
help="Disable multimodal input chunking attention for V1. "
1035
"If set to true and chunked prefill is enabled, we do not want to"
1036
" partially schedule a multimodal item. This ensures that if a "
vllm/v1/core/encoder_cache_manager.py
+8
@@ -133,6 +133,14 @@ def _compute_encoder_budget_multimodal(
133
_, max_tokens_per_mm_item = max(max_tokens_by_modality_dict.items(),
134
key=lambda item: item[1])
135
136
+ if (scheduler_config.disable_chunked_mm_input and max_tokens_per_mm_item
137
+ > scheduler_config.max_num_batched_tokens):
138
+ raise ValueError(
139
+ "Chunked MM input disabled but max_tokens_per_mm_item "
140
+ f"({max_tokens_per_mm_item}) is larger than max_num_batched_tokens"
141
+ f" ({scheduler_config.max_num_batched_tokens}). Please increase "
142
+ "max_num_batched_tokens.")
143
+
144
encoder_compute_budget = max(scheduler_config.max_num_encoder_input_tokens,
145
max_tokens_per_mm_item)
146
encoder_cache_size = max(scheduler_config.encoder_cache_size,