A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix attention
Woosuk Kwon committed
Feb 23, 2023 at 21:32 UTC
87e0bcd426c746a17229e8e292cc2997d0719e9b
1 file changed
+12
-7
cacheflow/models/attention.py
+12
-7
@@ -44,19 +44,18 @@ class OPTCacheFlowAttention(nn.Module):
44
45
# FIXME(woosuk): Replace the following with a custom op.
46
for i in range(input_metadata.num_generation_tokens):
47
- q = query[i]
47
+ q = query[i].unsqueeze(0)
48
block_table = block_tables[i]
49
context_len = int(input_metadata.context_lens[i])
50
+
51
keys = []
52
for j in range(context_len):
53
block_number = block_table[j // block_size]
54
block_offset = j % block_size
55
k = key_cache[block_number, :, :, block_offset, :]
55
- k = k.view(num_heads, head_size)
56
+ k = k.reshape(num_heads, head_size)
57
keys.append(k)
57
- keys = torch.stack(keys, dim=-1)
58
- logits = q @ keys
59
- attention_weights = torch.softmax(logits, dim=-1)
58
+ keys = torch.stack(keys, dim=0)
59
60
values = []
61
for j in range(context_len):
@@ -64,8 +63,14 @@ class OPTCacheFlowAttention(nn.Module):
63
block_offset = j % block_size
64
v = value_cache[block_number, :, block_offset, :]
65
values.append(v)
67
- values = torch.stack(values, dim=-1)
68
- out = attention_weights @ values
66
+ values = torch.stack(values, dim=0)
67
+
68
+ q = q.unsqueeze(0)
69
+ keys = keys.unsqueeze(0)
70
+ values = values.unsqueeze(0)
71
+ out = xops.memory_efficient_attention(
72
+ q, keys, values, scale=self.scale)
73
+ out = out.view(num_heads, head_size)
74
output[i].copy_(out, non_blocking=True)
75
76
def forward(