A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[Core] Whisper enable `FULL_DECODE_ONLY` CudaGraph (#30072)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Dec 10, 2025 at 15:14 UTC
c756fb678184b867ed94e5613a529198f1aee423
3 files changed
+31
-12
tests/models/multimodal/generation/test_whisper.py
+2
@@ -103,6 +103,8 @@ def run_test(
103
max_model_len=448,
104
tensor_parallel_size=tensor_parallel_size,
105
distributed_executor_backend=distributed_executor_backend,
106
+ # TODO (NickLucche) figure out output differences with non-eager and re-enable
107
+ enforce_eager=True,
108
) as vllm_model:
109
llm = vllm_model.llm
110
vllm/config/vllm.py
+19
-11
@@ -666,8 +666,9 @@ class VllmConfig:
666
667
default_config = OPTIMIZATION_LEVEL_TO_CONFIG[self.optimization_level]
668
self._apply_optimization_level_defaults(default_config)
669
+
670
if (
670
- self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
671
+ self.compilation_config.cudagraph_mode.requires_piecewise_compilation()
672
and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
673
):
674
logger.info(
@@ -692,22 +693,29 @@ class VllmConfig:
693
694
if current_platform.support_static_graph_mode():
695
# if cudagraph_mode has full cudagraphs, we need to check support
695
- if (
696
- self.compilation_config.cudagraph_mode.has_full_cudagraphs()
697
- and self.model_config is not None
698
- ):
699
- if self.model_config.pooler_config is not None:
696
+ if model_config := self.model_config:
697
+ if (
698
+ self.compilation_config.cudagraph_mode.has_full_cudagraphs()
699
+ and model_config.pooler_config is not None
700
+ ):
701
logger.warning_once(
702
"Pooling models do not support full cudagraphs. "
703
"Overriding cudagraph_mode to PIECEWISE."
704
)
705
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
705
- elif self.model_config.is_encoder_decoder:
706
- logger.warning_once(
707
- "Encoder-decoder models do not support full cudagraphs. "
708
- "Overriding cudagraph_mode to PIECEWISE."
706
+ elif (
707
+ model_config.is_encoder_decoder
708
+ and self.compilation_config.cudagraph_mode
709
+ not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL_DECODE_ONLY)
710
+ ):
711
+ logger.info_once(
712
+ "Encoder-decoder models do not support %s. "
713
+ "Overriding cudagraph_mode to FULL_DECODE_ONLY.",
714
+ self.compilation_config.cudagraph_mode.name,
715
+ )
716
+ self.compilation_config.cudagraph_mode = (
717
+ CUDAGraphMode.FULL_DECODE_ONLY
718
)
710
- self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
719
720
# disable cudagraph when enforce eager execution
721
if self.model_config is not None and self.model_config.enforce_eager:
vllm/v1/worker/gpu_model_runner.py
+10
-1
@@ -1267,6 +1267,8 @@ class GPUModelRunner(
1267
if not isinstance(kv_cache_spec, CrossAttentionSpec):
1268
return None, None
1269
1270
+ # Zero out buffer for padding requests that are not actually scheduled (CGs)
1271
+ self.encoder_seq_lens.np[:num_reqs] = 0
1272
# Build encoder_seq_lens array mapping request indices to
1273
# encoder lengths for inputs scheduled in this batch
1274
for req_id in num_scheduled_tokens:
@@ -2764,6 +2766,7 @@ class GPUModelRunner(
2766
# be improved in model runner v2)
2767
force_uniform_decode: bool | None = None,
2768
force_has_lora: bool | None = None,
2769
+ num_encoder_reqs: int = 0,
2770
) -> tuple[
2771
CUDAGraphMode,
2772
BatchDescriptor,
@@ -2780,6 +2783,11 @@ class GPUModelRunner(
2783
if force_uniform_decode is None
2784
else force_uniform_decode
2785
)
2786
+ # Encoder-decoder models only support CG for decoder_step > 0 (no enc_output
2787
+ # is present). Also, chunked-prefill is disabled, so batch are uniform.
2788
+ has_encoder_output = (
2789
+ self.model_config.is_encoder_decoder and num_encoder_reqs > 0
2790
+ )
2791
2792
has_lora = (
2793
len(self.input_batch.lora_id_to_lora_request) > 0
@@ -2799,7 +2807,7 @@ class GPUModelRunner(
2807
)
2808
2809
cudagraph_mode, batch_descriptor = dispatch_cudagraph(
2802
- num_tokens_padded, use_cascade_attn
2810
+ num_tokens_padded, use_cascade_attn or has_encoder_output
2811
)
2812
num_tokens_padded = batch_descriptor.num_tokens
2813
@@ -2997,6 +3005,7 @@ class GPUModelRunner(
3005
num_scheduled_tokens_np=num_scheduled_tokens_np,
3006
max_num_scheduled_tokens=max_num_scheduled_tokens,
3007
use_cascade_attn=cascade_attn_prefix_lens is not None,
3008
+ num_encoder_reqs=len(scheduler_output.scheduled_encoder_inputs),
3009
)
3010
3011
logger.debug(