A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
Fix CUDA graph decode capture crash in AITER FlashAttention (#36042)
Signed-off-by: Martin Yuan <myuan@meta.com> Co-authored-by: Martin Yuan <myuan@meta.com>
Mengtao (Martin) Yuan committed
Mar 6, 2026 at 18:12 UTC
1a9718085c7980443558db1ff4160c58096a3f0e
1 file changed
+3
-4
vllm/v1/attention/backends/rocm_aiter_fa.py
+3
-4
@@ -1152,11 +1152,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
1152
decode_max_query_len = attn_metadata.decode_metadata.max_query_len
1153
1154
# Use unified_attention for speculative decoding (multi-token)
1155
- # or when sliding window is enabled
1156
- if self.sliding_window[0] != -1 or decode_max_query_len > 1:
1155
+ if decode_max_query_len > 1:
1156
assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), (
1158
- "Shuffle KV cache layout is not supported with sliding "
1159
- "window or speculative decoding (multi-token decode)."
1157
+ "Shuffle KV cache layout is not supported with "
1158
+ "speculative decoding (multi-token decode)."
1159
)
1160
from aiter.ops.triton.unified_attention import (
1161
unified_attention,