A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Bugfix] Fix stale `common_attn_metadata.max_seq_len` in speculative decoding with Eagle (#32312)
Signed-off-by: Ofir Zafrir <ofir.zafrir@intel.com>
Ofir Zafrir committed
Jan 15, 2026 at 08:39 UTC
e9ec2a72d845e1f3374c6de68e361edc6258c891
1 file changed
+6
vllm/v1/spec_decode/eagle.py
+6
@@ -466,6 +466,12 @@ class EagleProposer:
466
# For the requests that exceed the max model length, we set the
467
# sequence length to 1 to minimize their overheads in attention.
468
common_attn_metadata.seq_lens.masked_fill_(exceeds_max_model_len, 1)
469
+ # Increment the maximum sequence length. We increment max_seq_len
470
+ # unconditionally even though some seq_lens may have been capped above,
471
+ # as max_seq_len serves as an upper bound for sequence lengths.
472
+ common_attn_metadata.max_seq_len = min(
473
+ common_attn_metadata.max_seq_len + 1, self.max_model_len
474
+ )
475
476
# Also update the CPU-side shadow; NOTE: this is hacky and should be
477
# removed in when common_attn_metadata.seq_lens_cpu is deprecated.