A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Remove xformers
Woosuk Kwon committed
Feb 24, 2023 at 08:36 UTC
7f22f90e8cb423fdaa35203d41badd734d9c2e86
1 file changed
+20
-16
cacheflow/models/attention.py
+20
-16
@@ -2,7 +2,6 @@ from typing import Optional
2
3
import torch
4
import torch.nn as nn
5
-import xformers.ops as xops
5
6
from cacheflow import ops
7
from cacheflow.models import InputMetadata
@@ -14,8 +13,20 @@ class OPTCacheFlowAttention(nn.Module):
13
super().__init__()
14
self.scale = scale
15
17
- # Shape-agnostic attention mask.
18
- self.attention_mask = xops.LowerTriangularMask()
16
+ def _masked_attention(
17
+ self,
18
+ query: torch.Tensor, # [num_queries, num_heads, head_size]
19
+ key: torch.Tensor, # [num_keys, num_heads, head_size]
20
+ value: torch.Tensor, # [num_keys, num_heads, head_size]
21
+ attn_mask: Optional[torch.Tensor] = None, # [num_queries, num_keys]
22
+ ) -> torch.Tensor: # [num_queries, num_heads, head_size]
23
+ query = query * self.scale
24
+ attn = torch.einsum('qhd,khd->hqk', query, key)
25
+ if attn_mask is not None:
26
+ attn = attn + attn_mask
27
+ attn = torch.softmax(attn, dim=-1)
28
+ out = torch.einsum('hqk,khd->qhd', attn, value)
29
+ return out
30
31
def multi_query_kv_attention(
32
self,
@@ -24,13 +35,11 @@ class OPTCacheFlowAttention(nn.Module):
35
key: torch.Tensor,
36
value: torch.Tensor,
37
) -> None:
27
- query = query.unsqueeze(0)
28
- key = key.unsqueeze(0)
29
- value = value.unsqueeze(0)
30
- out = xops.memory_efficient_attention(
31
- query, key, value, attn_bias=self.attention_mask, scale=self.scale)
32
- out = out.squeeze(0)
33
- # FIXME(woosuk): Directly write the attention output.
38
+ # FIXME(woosuk): Replace this with a custom op call.
39
+ attention_mask = torch.triu(
40
+ torch.ones(query.shape[0], key.shape[0]), diagonal=1) * -1e5
41
+ attention_mask = attention_mask.to(dtype=query.dtype, device=query.device)
42
+ out = self._masked_attention(query, key, value, attention_mask)
43
output.copy_(out, non_blocking=True)
44
45
def single_query_cached_kv_attention(
@@ -64,15 +73,10 @@ class OPTCacheFlowAttention(nn.Module):
73
74
v = value_cache[block_number, :, block_offset, :]
75
values.append(v)
67
-
76
keys = torch.stack(keys, dim=0)
77
values = torch.stack(values, dim=0)
78
71
- q = q.unsqueeze(0)
72
- keys = keys.unsqueeze(0)
73
- values = values.unsqueeze(0)
74
- out = xops.memory_efficient_attention(
75
- q, keys, values, scale=self.scale)
79
+ out = self._masked_attention(q, keys, values)
80
out = out.view(num_heads, head_size)
81
output[i].copy_(out, non_blocking=True)
82