@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75471386de
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

use flash-attn via xformers (#877)

Aman Gupta Karmani committed Aug 30, 2023 at 00:52 UTC 75471386de62eef044568bf4425b63d410fde11f
2 files changed -5
tests/kernels/test_attention.py
-2
@@ -266,7 +266,6 @@ def run_multi_query_kv_attention(
266 qkv.uniform_(-1e-3, 1e-3)
267 query, key, value = qkv.unbind(dim=1)
268
269 - attn_op = xops.fmha.cutlass.FwOp()
269 attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
270 output = xops.memory_efficient_attention_forward(
271 query.unsqueeze(0),
@@ -275,7 +274,6 @@ def run_multi_query_kv_attention(
274 attn_bias=attn_bias,
275 p=0.0,
276 scale=scale,
278 - op=attn_op,
277 )
278 output = output.squeeze(0)
279
vllm/model_executor/layers/attention.py
-3
@@ -61,7 +61,6 @@ class PagedAttention(nn.Module):
61 self.num_heads = num_heads
62 self.head_size = head_size
63 self.scale = float(scale)
64 - self.attn_op = xops.fmha.cutlass.FwOp()
64 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
65
66 assert self.num_heads % self.num_kv_heads == 0
@@ -115,7 +114,6 @@ class PagedAttention(nn.Module):
114 attn_bias=input_metadata.attn_bias[0],
115 p=0.0,
116 scale=self.scale,
118 - op=self.attn_op,
117 )
118 # TODO(woosuk): Unnecessary copy. Optimize.
119 output.copy_(out.squeeze(0))
@@ -404,7 +402,6 @@ class PagedAttentionWithALiBi(PagedAttention):
402 attn_bias=input_metadata.attn_bias[i],
403 p=0.0,
404 scale=self.scale,
407 - op=self.attn_op,
405 )
406 # TODO(woosuk): Unnecessary copy. Optimize.
407 output[start:end].copy_(out.squeeze(0))