@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a0309a646
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc][Bugfix] FA3 support to ViT MHA layer (#12435)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Roger Wang committed Jan 25, 2025 at 21:00 UTC 2a0309a646b1ed83a0c40974e08c8dc628726d3c
1 file changed +22 -3
vllm/attention/layer.py
+22 -3
@@ -251,9 +251,28 @@ class MultiHeadAttention(nn.Module):
251 _Backend.FLASH_ATTN,
252 _Backend.FLASH_ATTN_VLLM_V1,
253 }:
254 - from vllm.vllm_flash_attn import flash_attn_func
255 -
256 - out = flash_attn_func(query, key, value, softmax_scale=self.scale)
254 + from vllm.vllm_flash_attn import flash_attn_varlen_func
255 +
256 + cu_seqlens_q = torch.arange(0, (bsz + 1) * q_len,
257 + step=q_len,
258 + dtype=torch.int32,
259 + device=query.device)
260 + cu_seqlens_k = torch.arange(0, (bsz + 1) * kv_len,
261 + step=kv_len,
262 + dtype=torch.int32,
263 + device=key.device)
264 +
265 + out = flash_attn_varlen_func(
266 + query.flatten(0, 1),
267 + key.flatten(0, 1),
268 + value.flatten(0, 1),
269 + cu_seqlens_q=cu_seqlens_q,
270 + cu_seqlens_k=cu_seqlens_k,
271 + max_seqlen_q=q_len,
272 + max_seqlen_k=kv_len,
273 + softmax_scale=self.scale,
274 + )
275 + out = out.reshape(bsz, q_len, -1)
276 elif self.attn_backend == _Backend.XFORMERS:
277 from xformers import ops as xops
278