@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a5255270c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Revert FA on ViT #12355 and #12435 (#12445)

Roger Wang committed Jan 26, 2025 at 03:56 UTC a5255270c3ad492b5def19fe38beb9b2df30e74f
1 file changed +4 -37
vllm/attention/layer.py
+4 -37
@@ -210,9 +210,6 @@ class MultiHeadAttention(nn.Module):
210 self.scale = scale
211 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
212
213 - assert self.num_heads % self.num_kv_heads == 0
214 - self.num_queries_per_kv = self.num_heads // self.num_kv_heads
215 -
213 dtype = torch.get_default_dtype()
214 attn_backend = get_attn_backend(head_size,
215 dtype,
@@ -220,12 +217,12 @@ class MultiHeadAttention(nn.Module):
217 block_size=16,
218 is_attention_free=False)
219 backend = backend_name_to_enum(attn_backend.get_name())
220 + if backend in {_Backend.FLASH_ATTN, _Backend.FLASH_ATTN_VLLM_V1}:
221 + backend = _Backend.XFORMERS
222
223 self.attn_backend = backend if backend in {
224 _Backend.TORCH_SDPA,
225 _Backend.XFORMERS,
227 - _Backend.FLASH_ATTN,
228 - _Backend.FLASH_ATTN_VLLM_V1,
226 } else _Backend.TORCH_SDPA
227
228 def forward(
@@ -235,6 +232,7 @@ class MultiHeadAttention(nn.Module):
232 value: torch.Tensor,
233 ) -> torch.Tensor:
234 """Input shape: batch_size x seq_len x hidden_size"""
235 + # TODO(Isotr0py): Use existing backend implementations and support FA3
236 bsz, q_len, _ = query.size()
237 kv_len = key.size(1)
238
@@ -242,38 +240,7 @@ class MultiHeadAttention(nn.Module):
240 key = key.view(bsz, kv_len, self.num_kv_heads, self.head_size)
241 value = value.view(bsz, kv_len, self.num_kv_heads, self.head_size)
242
245 - if (num_repeat := self.num_queries_per_kv) > 1:
246 - # Handle MQA and GQA
247 - key = torch.repeat_interleave(key, num_repeat, dim=2)
248 - value = torch.repeat_interleave(value, num_repeat, dim=2)
249 -
250 - if self.attn_backend in {
251 - _Backend.FLASH_ATTN,
252 - _Backend.FLASH_ATTN_VLLM_V1,
253 - }:
254 - from vllm.vllm_flash_attn import flash_attn_varlen_func
255 -
256 - cu_seqlens_q = torch.arange(0, (bsz + 1) * q_len,
257 - step=q_len,
258 - dtype=torch.int32,
259 - device=query.device)
260 - cu_seqlens_k = torch.arange(0, (bsz + 1) * kv_len,
261 - step=kv_len,
262 - dtype=torch.int32,
263 - device=key.device)
264 -
265 - out = flash_attn_varlen_func(
266 - query.flatten(0, 1),
267 - key.flatten(0, 1),
268 - value.flatten(0, 1),
269 - cu_seqlens_q=cu_seqlens_q,
270 - cu_seqlens_k=cu_seqlens_k,
271 - max_seqlen_q=q_len,
272 - max_seqlen_k=kv_len,
273 - softmax_scale=self.scale,
274 - )
275 - out = out.reshape(bsz, q_len, -1)
276 - elif self.attn_backend == _Backend.XFORMERS:
243 + if self.attn_backend == _Backend.XFORMERS:
244 from xformers import ops as xops
245
246 out = xops.memory_efficient_attention_forward(query,