@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 87e0bcd426
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix attention

Woosuk Kwon committed Feb 23, 2023 at 21:32 UTC 87e0bcd426c746a17229e8e292cc2997d0719e9b
1 file changed +12 -7
cacheflow/models/attention.py
+12 -7
@@ -44,19 +44,18 @@ class OPTCacheFlowAttention(nn.Module):
44
45 # FIXME(woosuk): Replace the following with a custom op.
46 for i in range(input_metadata.num_generation_tokens):
47 - q = query[i]
47 + q = query[i].unsqueeze(0)
48 block_table = block_tables[i]
49 context_len = int(input_metadata.context_lens[i])
50 +
51 keys = []
52 for j in range(context_len):
53 block_number = block_table[j // block_size]
54 block_offset = j % block_size
55 k = key_cache[block_number, :, :, block_offset, :]
55 - k = k.view(num_heads, head_size)
56 + k = k.reshape(num_heads, head_size)
57 keys.append(k)
57 - keys = torch.stack(keys, dim=-1)
58 - logits = q @ keys
59 - attention_weights = torch.softmax(logits, dim=-1)
58 + keys = torch.stack(keys, dim=0)
59
60 values = []
61 for j in range(context_len):
@@ -64,8 +63,14 @@ class OPTCacheFlowAttention(nn.Module):
63 block_offset = j % block_size
64 v = value_cache[block_number, :, block_offset, :]
65 values.append(v)
67 - values = torch.stack(values, dim=-1)
68 - out = attention_weights @ values
66 + values = torch.stack(values, dim=0)
67 +
68 + q = q.unsqueeze(0)
69 + keys = keys.unsqueeze(0)
70 + values = values.unsqueeze(0)
71 + out = xops.memory_efficient_attention(
72 + q, keys, values, scale=self.scale)
73 + out = out.view(num_heads, head_size)
74 output[i].copy_(out, non_blocking=True)
75
76 def forward(