A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Change default CUDAGraphMode from PIECEWISE to FULL_AND_PIECEWISE (#25444)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Sep 23, 2025 at 15:29 UTC
24fab45d96a91b491db338ee02cd24e55b7fbb5f
3 files changed
+32
-7
vllm/config/__init__.py
+8
-1
@@ -509,8 +509,15 @@ class VllmConfig:
509
if self.compilation_config.cudagraph_mode is None:
510
if envs.VLLM_USE_V1 and self.compilation_config.level \
511
== CompilationLevel.PIECEWISE:
512
+ # default to full and piecewise for most models
513
self.compilation_config.cudagraph_mode = \
513
- CUDAGraphMode.PIECEWISE
514
+ CUDAGraphMode.FULL_AND_PIECEWISE
515
+
516
+ # pooling model does not support full cudagraphs
517
+ if self.model_config is not None and \
518
+ self.model_config.pooler_config is not None:
519
+ self.compilation_config.cudagraph_mode = \
520
+ CUDAGraphMode.PIECEWISE
521
else:
522
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
523
vllm/config/compilation.py
+3
-4
@@ -228,15 +228,14 @@ class CompilationConfig:
228
The mode of the cudagraph:
229
230
- NONE, no cudagraph capture.
231
- - PIECEWISE. (v1 default)
231
+ - PIECEWISE.
232
- FULL.
233
- FULL_DECODE_ONLY.
234
- - FULL_AND_PIECEWISE.
234
+ - FULL_AND_PIECEWISE. (v1 default)
235
236
PIECEWISE mode build piecewise cudagraph only, keeping the cudagraph
237
incompatible ops (i.e. some attention ops) outside the cudagraph
238
for general flexibility.
239
- This is the default mode.
239
240
FULL mode: Capture full cudagraph for all batches. Can be good for small
241
models or workloads with small prompts; not supported by many backends.
@@ -249,7 +248,7 @@ class CompilationConfig:
248
249
FULL_AND_PIECEWISE mode: Capture full cudagraph for decode batches and
250
piecewise cudagraph for prefill and mixed prefill-decode batches.
252
- This is like the most performant mode for most models.
251
+ This is the most performant mode for most models and is the default.
252
253
Currently, the cudagraph mode is only used for the v1 engine.
254
Note that the cudagraph logic is generally orthogonal to the
vllm/v1/worker/gpu_model_runner.py
+21
-2
@@ -2947,8 +2947,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2947
# TODO(luka) better system for describing dummy batches
2948
seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1]
2949
else:
2950
- # Make sure max_model_len is used at the graph capture time.
2951
- seq_lens = self.max_model_len
2950
+ seq_lens = max_query_len
2951
self.seq_lens.np[:num_reqs] = seq_lens
2952
self.seq_lens.np[num_reqs:] = 0
2953
self.seq_lens.copy_to_gpu()
@@ -3541,6 +3540,26 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3540
CUDAGraphMode.FULL_DECODE_ONLY
3541
logger.warning(msg)
3542
3543
+ # check that if we are doing decode full-cudagraphs it is supported
3544
+ if (cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
3545
+ and min_cg_support == AttentionCGSupport.NEVER):
3546
+ msg = (f"CUDAGraphMode.{cudagraph_mode.name} is not supported "
3547
+ f"with {min_cg_builder_name} backend (support: "
3548
+ f"{min_cg_support})")
3549
+ if (self.compilation_config.level == CompilationLevel.PIECEWISE and
3550
+ (self.compilation_config.splitting_ops_contain_attention()
3551
+ or self.compilation_config.use_inductor_graph_partition)):
3552
+ msg += "; setting cudagraph_mode=PIECEWISE because "\
3553
+ "attention is compiled piecewise"
3554
+ cudagraph_mode = self.compilation_config.cudagraph_mode = \
3555
+ CUDAGraphMode.PIECEWISE
3556
+ else:
3557
+ msg += "; setting cudagraph_mode=NONE because "\
3558
+ "attention is not compiled piecewise"
3559
+ cudagraph_mode = self.compilation_config.cudagraph_mode = \
3560
+ CUDAGraphMode.NONE
3561
+ logger.warning(msg)
3562
+
3563
# check that if we are doing spec-decode + decode full-cudagraphs it is
3564
# supported
3565
if (cudagraph_mode.decode_mode() == CUDAGraphMode.FULL