A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 07:15 UTC
[crashfix] Eagle + multimodal can crash on mm cache miss (#29750)
Signed-off-by: Mickael Seznec <mickael@mistral.ai> Co-authored-by: Roger Wang <hey@rogerw.io>
Mickaël Seznec committed
Dec 1, 2025 at 10:29 UTC
86e178f7c4d8c3b0eaf3c8e3f810a83f63b90e24
1 file changed
+11
-3
vllm/v1/core/sched/scheduler.py
+11
-3
@@ -263,6 +263,7 @@ class Scheduler(SchedulerInterface):
263
request.num_computed_tokens,
264
num_new_tokens,
265
encoder_compute_budget,
266
+ shift_computed_tokens=1 if self.use_eagle else 0,
267
)
268
269
if num_new_tokens == 0:
@@ -532,6 +533,7 @@ class Scheduler(SchedulerInterface):
533
num_computed_tokens,
534
num_new_tokens,
535
encoder_compute_budget,
536
+ shift_computed_tokens=1 if self.use_eagle else 0,
537
)
538
if num_new_tokens == 0:
539
# The request cannot be scheduled.
@@ -829,6 +831,7 @@ class Scheduler(SchedulerInterface):
831
num_computed_tokens: int,
832
num_new_tokens: int,
833
encoder_compute_budget: int,
834
+ shift_computed_tokens: int = 0,
835
) -> tuple[list[int], int, int, list[int]]:
836
"""
837
Determine which encoder inputs need to be scheduled in the current step,
@@ -873,7 +876,10 @@ class Scheduler(SchedulerInterface):
876
# The encoder output is needed if the two ranges overlap:
877
# [num_computed_tokens, num_computed_tokens + num_new_tokens) and
878
# [start_pos, start_pos + num_encoder_tokens)
876
- if start_pos >= num_computed_tokens + num_new_tokens:
879
+ if (
880
+ start_pos
881
+ >= num_computed_tokens + num_new_tokens + shift_computed_tokens
882
+ ):
883
# The encoder input is not needed in this step.
884
break
885
@@ -929,10 +935,12 @@ class Scheduler(SchedulerInterface):
935
# NOTE(woosuk): We assume that the encoder input tokens should
936
# be processed altogether, as the encoder usually uses
937
# bidirectional attention.
932
- if num_computed_tokens < start_pos:
938
+ if num_computed_tokens + shift_computed_tokens < start_pos:
939
# We only schedule the decoder tokens just before the
940
# encoder input.
935
- num_new_tokens = start_pos - num_computed_tokens
941
+ num_new_tokens = start_pos - (
942
+ num_computed_tokens + shift_computed_tokens
943
+ )
944
else:
945
# Because of prefix caching, num_computed_tokens is greater
946
# than start_pos even though its encoder input is not