A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 14:13 UTC
[V1][BugFix] Fix EAGLE3 encoder cache miss with disable_chunked_mm_input (#34220)
Signed-off-by: KrxGu <krishom70@gmail.com>
Krish Gupta committed
Feb 10, 2026 at 18:35 UTC
748625cdafd7898b163115d8c33c7c5521a708e8
2 files changed
+75
-1
tests/v1/core/test_scheduler.py
+69
@@ -3675,3 +3675,72 @@ def test_abort_request_finished_recving():
3675
# verify request is deleted
3676
assert request.request_id not in scheduler.requests
3677
assert not scheduler.finished_recving_kv_req_ids
3678
+
3679
+
3680
+def test_eagle3_mm_encoder_cache_with_shift():
3681
+ """Test EAGLE3 encoder scheduling accounts for shift_computed_tokens.
3682
+
3683
+ Regression test for issue #32469: When EAGLE3 is enabled with
3684
+ disable_chunked_mm_input=True, ensure encoder inputs are scheduled
3685
+ when tokens overlap the MM range, properly accounting for
3686
+ shift_computed_tokens in the boundary calculation.
3687
+
3688
+ Without the fix, the scheduler would fail to schedule encoder inputs
3689
+ at the boundary, causing "Encoder cache miss" errors.
3690
+ """
3691
+ scheduler = create_scheduler(
3692
+ model="llava-hf/llava-1.5-7b-hf",
3693
+ max_num_batched_tokens=1024,
3694
+ disable_chunked_mm_input=True,
3695
+ max_model_len=2048,
3696
+ num_speculative_tokens=4, # This enables EAGLE with shift=1
3697
+ )
3698
+
3699
+ mm_start_pos = 100
3700
+ mm_length = 576
3701
+
3702
+ mm_positions = [
3703
+ [PlaceholderRange(offset=mm_start_pos, length=mm_length)],
3704
+ ]
3705
+
3706
+ requests = create_requests(
3707
+ num_requests=1,
3708
+ num_tokens=mm_start_pos + mm_length + 100,
3709
+ mm_positions=mm_positions,
3710
+ )
3711
+
3712
+ # Start with some tokens already computed to simulate decoding
3713
+ request = requests[0]
3714
+ request.num_computed_tokens = 0
3715
+
3716
+ scheduler.add_request(request)
3717
+ output = scheduler.schedule()
3718
+
3719
+ assert output is not None
3720
+ shift_computed_tokens = 1
3721
+ req_id = request.request_id
3722
+
3723
+ assert req_id in output.num_scheduled_tokens
3724
+ num_scheduled = output.num_scheduled_tokens[req_id]
3725
+
3726
+ mm_feature = request.mm_features[0]
3727
+ start_pos = mm_feature.mm_position.offset
3728
+ tokens_end = request.num_computed_tokens + num_scheduled
3729
+ scheduled_end_with_shift = tokens_end + shift_computed_tokens
3730
+
3731
+ # Assert that we scheduled into the MM range (test setup verification)
3732
+ assert scheduled_end_with_shift > start_pos, (
3733
+ f"Test setup error: expected to schedule into MM range. "
3734
+ f"scheduled_end_with_shift={scheduled_end_with_shift}, "
3735
+ f"start_pos={start_pos}"
3736
+ )
3737
+
3738
+ # The key assertion: when scheduled tokens overlap MM range
3739
+ # (accounting for EAGLE's shift), encoder MUST be scheduled.
3740
+ # Without the fix, this would fail at the boundary case.
3741
+ assert req_id in output.scheduled_encoder_inputs, (
3742
+ f"Encoder input missing: scheduled {num_scheduled} tokens "
3743
+ f"(computed={request.num_computed_tokens}, end={tokens_end}, "
3744
+ f"shifted_end={scheduled_end_with_shift}) overlapping MM at "
3745
+ f"{start_pos}. The fix must schedule encoder inputs."
3746
+ )
vllm/v1/core/sched/scheduler.py
+6
-1
@@ -1155,7 +1155,12 @@ class Scheduler(SchedulerInterface):
1155
and (num_computed_tokens + num_new_tokens)
1156
< (start_pos + num_encoder_tokens)
1157
):
1158
- num_new_tokens = start_pos - num_computed_tokens
1158
+ # Account for EAGLE shift when rolling back to avoid
1159
+ # encoder cache miss. This ensures the scheduled range
1160
+ # stops before start_pos even with the shift.
1161
+ num_new_tokens = max(
1162
+ 0, start_pos - (num_computed_tokens + shift_computed_tokens)
1163
+ )
1164
break
1165
if not self.encoder_cache_manager.can_allocate(
1166
request, i, encoder_compute_budget, num_embeds_to_schedule