[Bugfix][Core] Schedule encoder-only prompts larger than one step (#59029)
Signed-off-by: Giulio De Pasquale <git@depasquale.giugl.io> Co-authored-by: Isotr0py <2037008807@qq.com>
Giulio De Pasquale committed
Sep 30, 2026 at 15:04 UTC
2e52a558f04307b3f6f842e937e35aac079121c1
3 files changed
+50
tests/v1/core/test_scheduler.py
+46
@@ -437,6 +437,52 @@ def test_schedule_partial_requests():
437
assert requests[2].request_id not in output.num_scheduled_tokens
438
439
440
+def test_encoder_only_prompt_longer_than_budget_is_chunked():
441
+ """The engine switches chunked prefill off for an encoder-only instance,
442
+ but the token budget must still split its prompt across steps instead of
443
+ leaving the request waiting forever."""
444
+ scheduler = create_scheduler(
445
+ model="llava-hf/llava-1.5-7b-hf",
446
+ max_num_batched_tokens=1024,
447
+ max_model_len=4096,
448
+ mm_encoder_only=True,
449
+ )
450
+ # EngineCore turns this off after config validation whenever the instance
451
+ # holds no KV cache, which is every encoder-only instance.
452
+ scheduler.scheduler_config.enable_chunked_prefill = False
453
+ (request,) = create_requests(
454
+ num_requests=1,
455
+ num_tokens=2500,
456
+ mm_positions=[[PlaceholderRange(offset=100, length=600)]],
457
+ )
458
+ scheduler.add_request(request)
459
+
460
+ def advance(output):
461
+ scheduler.update_from_output(
462
+ output,
463
+ ModelRunnerOutput(
464
+ req_ids=[request.request_id],
465
+ req_id_to_index={request.request_id: 0},
466
+ sampled_token_ids=[[]],
467
+ logprobs=None,
468
+ prompt_logprobs_dict={},
469
+ pooler_output=[],
470
+ ),
471
+ )
472
+
473
+ first = scheduler.schedule()
474
+ assert first.num_scheduled_tokens[request.request_id] == 1024
475
+ assert request.request_id in first.scheduled_encoder_inputs
476
+ advance(first)
477
+
478
+ second = scheduler.schedule()
479
+ assert second.num_scheduled_tokens[request.request_id] == 1024
480
+ advance(second)
481
+
482
+ third = scheduler.schedule()
483
+ assert third.num_scheduled_tokens[request.request_id] == 452
484
+
485
+
486
@pytest.mark.parametrize("has_running", [True, False])
487
def test_schedule_prefills_gating(has_running: bool):
488
"""DP prefill-balancing gate: when `throttle_prefills` is True, a new
tests/v1/core/utils.py
+3
@@ -60,6 +60,7 @@ def create_scheduler(
60
long_prefill_token_threshold: int = 0,
61
long_prefill_token_threshold_adaptive: bool = False,
62
disable_chunked_mm_input: bool = False,
63
+ mm_encoder_only: bool = False,
64
use_kv_connector: None | bool | str | MockKVConfig = None,
65
kv_role: str = "kv_both",
66
num_blocks: int = 10000,
@@ -109,6 +110,8 @@ def create_scheduler(
110
# SchedulerConfig one, so both must agree.
111
max_model_len=max_model_len,
112
)
113
+ if mm_encoder_only:
114
+ model_config.multimodal_config.mm_encoder_only = True
115
if use_ec_connector and ec_role == "ec_producer":
116
model_config.multimodal_config = MultiModalConfig()
117
if max_model_len is None:
vllm/v1/core/sched/scheduler.py
+1
@@ -1115,6 +1115,7 @@ class Scheduler(SchedulerInterface):
1115
if (
1116
not self.scheduler_config.enable_chunked_prefill
1117
and num_new_tokens > request_token_budget
1118
+ and not self.is_mm_encoder_only
1119
):
1120
# If chunked_prefill is disabled,
1121
# we can stop the scheduling here.