A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Model][AMD] ROCm support for 256 head dims for Gemma (#3972)
James Whedbee committed
Apr 10, 2024 at 10:12 UTC
8b317c6dd09ce566f4b4abeb446585ac75262cce
1 file changed
+2
-3
vllm/attention/ops/triton_flash_attention.py
+2
-3
@@ -677,8 +677,7 @@ def check_args(
677
assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1]
678
# TODO: Change assert if we support qkl f8 and v f16
679
assert q.dtype == k.dtype and q.dtype == v.dtype
680
- # TODO: Fix assert to check head size <=256 once supported
681
- assert head_size <= 128
680
+ assert head_size <= 256
681
assert o.shape == q.shape
682
assert (nheads_q % nheads_k) == 0
683
@@ -729,7 +728,7 @@ class _attention(torch.autograd.Function):
728
o_strides = (o.stride(0), o.stride(2), o.stride(1), o.stride(3))
729
730
# Get closest power of 2 over or equal to 32.
732
- unpadded_head_dims = {32, 64, 128}
731
+ unpadded_head_dims = {32, 64, 128, 256}
732
if head_size not in unpadded_head_dims:
733
padded_d_model = None
734
for i in unpadded_head_dims: