A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[Bugfix] Add Multiple of 16 block_size to triton fallback on rocm Attention to support qwen3_5 (#35923)
Signed-off-by: JartX <sagformas@epdcenter.es> Co-authored-by: akaratza <akaratza@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
JartX committed
Mar 11, 2026 at 08:45 UTC
a40ee486f273eaaa885dafd0526f42f3a5b960c9
2 files changed
+11
-23
docs/design/attention_backends.md
+1
-1
@@ -173,7 +173,7 @@ Priority is **1 = highest** (tried first).
173
| `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `bfloat16` | Any | Any | ❌ | ✅ | ❌ | Decoder, Encoder Only | Any |
174
| `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ❌ | ❌ | ❌ | Decoder, Enc-Dec | N/A |
175
| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto` | %16 | Any | ✅ | ✅ | ❌ | All | N/A |
176
-| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 544 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ✅ | ✅ | ❌ | All | N/A |
176
+| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ✅ | ✅ | ❌ | All | N/A |
177
| `TREE_ATTN` | | fp16, bf16 | `auto` | %16 | 32, 64, 96, 128, 160, 192, 224, 256 | ❌ | ❌ | ❌ | Decoder | Any |
178
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ✅ | ❌ | All | Any |
179
vllm/v1/attention/backends/rocm_attn.py
+10
-22
@@ -174,25 +174,15 @@ class RocmAttentionBackend(AttentionBackend):
174
175
@staticmethod
176
def get_supported_kernel_block_sizes() -> list[int | MultipleOf]:
177
- # ROCM paged attention kernel only supports block sizes 16 and 32
177
+ # ROCM paged attention native C++ kernel only supports block sizes 16 and 32
178
# due to shared memory (LDS) constraints on AMD GPUs.
179
# See csrc/rocm/attention.cu CALL_CUSTOM_LAUNCHER_BLK macro.
180
-
181
- # However, The limitations in [16, 32] are reasonable for a native C++ kernel,
182
- # but vLLM should allow support for non-standard sizes via the Triton path,
183
- # as addressed in this PR: https://github.com/vllm-project/vllm/pull/31380,
184
- # where the Triton kernel under rocm_atten does not support inference
185
- # for a non-standard qwen3-next model with a block_size of 544.
186
- # We have fixed the Triton kernel so that the standard model uses the original
187
- # bit-addressing logic, while the non-standard model
188
- # uses our optimized kernel logic.
189
- return [16, 32, 544]
190
-
191
- @classmethod
192
- def supports_block_size(cls, block_size: int | None) -> bool:
193
- if block_size is None:
194
- return True
195
- return block_size in (16, 32, 544)
180
+ # However, vLLM allows support for any multiple of 16 via the Triton path.
181
+ # As addressed in PR: https://github.com/vllm-project/vllm/pull/31380,
182
+ # non-standard models (like qwen3-next with block_size 544, or qwen3_5
183
+ # with 784 and 1056) are dynamically routed to our optimized Triton kernel
184
+ # in `do_kv_cache_update`.
185
+ return [MultipleOf(16)]
186
187
@classmethod
188
def get_supported_head_sizes(cls) -> list[int]:
@@ -463,11 +453,9 @@ class RocmAttentionImpl(AttentionImpl):
453
# Get the actual block_size from value_cache
454
# value_cache shape: [num_blocks, num_heads, head_size, block_size]
455
block_size = value_cache.shape[3]
466
- # Determine if it is a power of 2
467
- is_pow2 = block_size > 0 and (block_size & (block_size - 1) == 0)
456
469
- if is_pow2:
470
- # Normal 16, 32, 64, etc., use vLLM native HIP C++ logic
457
+ if block_size in (16, 32):
458
+ # Normal 16, 32, use vLLM native HIP C++ logic
459
PagedAttention.write_to_paged_cache(
460
key,
461
value,
@@ -479,7 +467,7 @@ class RocmAttentionImpl(AttentionImpl):
467
layer._v_scale,
468
)
469
else:
482
- # Case B: Non-standard blocks (e.g., 544 in Qwen3),
470
+ # Case B: Non-standard blocks (e.g., 64, 128, 544 in Qwen3Next or Qwen3.5 ),
471
# force using our modified Triton logic
472
triton_reshape_and_cache_flash(
473
key,