A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
use flash-attn via xformers (#877)
Aman Gupta Karmani committed
Aug 30, 2023 at 00:52 UTC
75471386de62eef044568bf4425b63d410fde11f
2 files changed
-5
tests/kernels/test_attention.py
-2
@@ -266,7 +266,6 @@ def run_multi_query_kv_attention(
266
qkv.uniform_(-1e-3, 1e-3)
267
query, key, value = qkv.unbind(dim=1)
268
269
- attn_op = xops.fmha.cutlass.FwOp()
269
attn_bias = BlockDiagonalCausalMask.from_seqlens(seq_lens)
270
output = xops.memory_efficient_attention_forward(
271
query.unsqueeze(0),
@@ -275,7 +274,6 @@ def run_multi_query_kv_attention(
274
attn_bias=attn_bias,
275
p=0.0,
276
scale=scale,
278
- op=attn_op,
277
)
278
output = output.squeeze(0)
279
vllm/model_executor/layers/attention.py
-3
@@ -61,7 +61,6 @@ class PagedAttention(nn.Module):
61
self.num_heads = num_heads
62
self.head_size = head_size
63
self.scale = float(scale)
64
- self.attn_op = xops.fmha.cutlass.FwOp()
64
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
65
66
assert self.num_heads % self.num_kv_heads == 0
@@ -115,7 +114,6 @@ class PagedAttention(nn.Module):
114
attn_bias=input_metadata.attn_bias[0],
115
p=0.0,
116
scale=self.scale,
118
- op=self.attn_op,
117
)
118
# TODO(woosuk): Unnecessary copy. Optimize.
119
output.copy_(out.squeeze(0))
@@ -404,7 +402,6 @@ class PagedAttentionWithALiBi(PagedAttention):
402
attn_bias=input_metadata.attn_bias[i],
403
p=0.0,
404
scale=self.scale,
407
- op=self.attn_op,
405
)
406
# TODO(woosuk): Unnecessary copy. Optimize.
407
output[start:end].copy_(out.squeeze(0))