@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8b317c6dd0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Model][AMD] ROCm support for 256 head dims for Gemma (#3972)

James Whedbee committed Apr 10, 2024 at 10:12 UTC 8b317c6dd09ce566f4b4abeb446585ac75262cce
1 file changed +2 -3
vllm/attention/ops/triton_flash_attention.py
+2 -3
@@ -677,8 +677,7 @@ def check_args(
677 assert q.shape[-1] == k.shape[-1] and q.shape[-1] == v.shape[-1]
678 # TODO: Change assert if we support qkl f8 and v f16
679 assert q.dtype == k.dtype and q.dtype == v.dtype
680 - # TODO: Fix assert to check head size <=256 once supported
681 - assert head_size <= 128
680 + assert head_size <= 256
681 assert o.shape == q.shape
682 assert (nheads_q % nheads_k) == 0
683
@@ -729,7 +728,7 @@ class _attention(torch.autograd.Function):
728 o_strides = (o.stride(0), o.stride(2), o.stride(1), o.stride(3))
729
730 # Get closest power of 2 over or equal to 32.
732 - unpadded_head_dims = {32, 64, 128}
731 + unpadded_head_dims = {32, 64, 128, 256}
732 if head_size not in unpadded_head_dims:
733 padded_d_model = None
734 for i in unpadded_head_dims: