A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Force PIECEWISE CUDAGraph mode for encoder-decoder (#25701)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Sep 25, 2025 at 21:21 UTC
13dd93c66766efff810feed3edb919f69d0d3836
1 file changed
+4
-2
vllm/config/__init__.py
+4
-2
@@ -364,9 +364,11 @@ class VllmConfig:
364
self.compilation_config.cudagraph_mode = \
365
CUDAGraphMode.FULL_AND_PIECEWISE
366
367
- # pooling model does not support full cudagraphs
367
+ # pooling models and encoder-decoder models
368
+ # do not support full cudagraphs
369
if self.model_config is not None and \
369
- self.model_config.pooler_config is not None:
370
+ (self.model_config.pooler_config is not None
371
+ or self.model_config.is_encoder_decoder):
372
self.compilation_config.cudagraph_mode = \
373
CUDAGraphMode.PIECEWISE
374
else: