A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 13:33 UTC
[Bugfix] Disable CG for Whisper+FA2 (#33164)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jan 27, 2026 at 14:46 UTC
1f3a2c2944f240ff043c3b7e01c4ae81461b09ed
1 file changed
+20
vllm/v1/attention/backends/flash_attn.py
+20
@@ -257,6 +257,26 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
257
)
258
supports_update_block_table: bool = True
259
260
+ @classmethod
261
+ def get_cudagraph_support(
262
+ cls,
263
+ vllm_config: "VllmConfig",
264
+ kv_cache_spec: "AttentionSpec",
265
+ ) -> AttentionCGSupport:
266
+ # FA2 does not support CUDA graphs with encoder-decoder models due to
267
+ # accuracy issues reported in https://github.com/vllm-project/vllm/issues/33091
268
+ if (
269
+ vllm_config.model_config.is_encoder_decoder
270
+ and get_flash_attn_version() == 2
271
+ ):
272
+ logger.warning_once(
273
+ "FlashAttention2 does not support CUDA graphs with "
274
+ "encoder-decoder models due to accuracy issues reported in #33091. "
275
+ "Disabling CUDA graph."
276
+ )
277
+ return AttentionCGSupport.NEVER
278
+ return cls._cudagraph_support
279
+
280
def __init__(
281
self,
282
kv_cache_spec: AttentionSpec,