A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[perf] Use CPU tensor to reduce GPU->CPU sync (#25884)
Signed-off-by: Lehua Ding <lehuading@tencent.com>
Lehua Ding committed
Sep 30, 2025 at 19:51 UTC
e184c9c510bca80c72cac06c9b041dcd90b4996e
1 file changed
+1
-1
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -2478,7 +2478,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2478
effective_drafter_max_model_len = (
2479
self.speculative_config.draft_model_config.max_model_len)
2480
input_fits_in_drafter = spec_decode_common_attn_metadata and (
2481
- spec_decode_common_attn_metadata.seq_lens.max() +
2481
+ spec_decode_common_attn_metadata.max_seq_len +
2482
self.speculative_config.num_speculative_tokens
2483
<= effective_drafter_max_model_len)
2484
if use_padded_batch_for_eagle and input_fits_in_drafter: