A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BUGFIX] Fix crash in Eagle Speculative Decoding models when exceedin… (#24662)
Signed-off-by: AlonKejzman <alonkeizman@gmail.com>
AlonKejzman committed
Sep 25, 2025 at 18:40 UTC
e04a1b6b21e1c410abbd6fa87880d5aa0c4277c2
1 file changed
+16
-2
vllm/v1/worker/gpu_model_runner.py
+16
-2
@@ -2310,7 +2310,20 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2310
use_padded_batch_for_eagle = self.speculative_config and \
2311
self.speculative_config.use_eagle() and \
2312
not self.speculative_config.disable_padded_drafter_batch
2313
- if use_padded_batch_for_eagle:
2313
+ effective_drafter_max_model_len = self.max_model_len
2314
+ if effective_drafter_max_model_len is None:
2315
+ effective_drafter_max_model_len = self.model_config.max_model_len
2316
+ if (self.speculative_config
2317
+ and self.speculative_config.draft_model_config is not None
2318
+ and self.speculative_config.draft_model_config.max_model_len
2319
+ is not None):
2320
+ effective_drafter_max_model_len = (
2321
+ self.speculative_config.draft_model_config.max_model_len)
2322
+ input_fits_in_drafter = spec_decode_common_attn_metadata and (
2323
+ spec_decode_common_attn_metadata.seq_lens.max() +
2324
+ self.speculative_config.num_speculative_tokens
2325
+ <= effective_drafter_max_model_len)
2326
+ if use_padded_batch_for_eagle and input_fits_in_drafter:
2327
# EAGLE speculative decoding can use the GPU sampled tokens
2328
# as inputs, and does not need to wait for bookkeeping to finish.
2329
propose_draft_token_ids(sampler_output.sampled_token_ids)
@@ -2328,7 +2341,8 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2341
logits, hidden_states,
2342
num_scheduled_tokens)
2343
2331
- if self.speculative_config and not use_padded_batch_for_eagle:
2344
+ if (self.speculative_config and not use_padded_batch_for_eagle
2345
+ and input_fits_in_drafter):
2346
# ngram and other speculative decoding methods use the sampled
2347
# tokens on the CPU, so they are run after bookkeeping.
2348
propose_draft_token_ids(valid_sampled_token_ids)