@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f22f90e8c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Remove xformers

Woosuk Kwon committed Feb 24, 2023 at 08:36 UTC 7f22f90e8cb423fdaa35203d41badd734d9c2e86
1 file changed +20 -16
cacheflow/models/attention.py
+20 -16
@@ -2,7 +2,6 @@ from typing import Optional
2
3 import torch
4 import torch.nn as nn
5 -import xformers.ops as xops
5
6 from cacheflow import ops
7 from cacheflow.models import InputMetadata
@@ -14,8 +13,20 @@ class OPTCacheFlowAttention(nn.Module):
13 super().__init__()
14 self.scale = scale
15
17 - # Shape-agnostic attention mask.
18 - self.attention_mask = xops.LowerTriangularMask()
16 + def _masked_attention(
17 + self,
18 + query: torch.Tensor, # [num_queries, num_heads, head_size]
19 + key: torch.Tensor, # [num_keys, num_heads, head_size]
20 + value: torch.Tensor, # [num_keys, num_heads, head_size]
21 + attn_mask: Optional[torch.Tensor] = None, # [num_queries, num_keys]
22 + ) -> torch.Tensor: # [num_queries, num_heads, head_size]
23 + query = query * self.scale
24 + attn = torch.einsum('qhd,khd->hqk', query, key)
25 + if attn_mask is not None:
26 + attn = attn + attn_mask
27 + attn = torch.softmax(attn, dim=-1)
28 + out = torch.einsum('hqk,khd->qhd', attn, value)
29 + return out
30
31 def multi_query_kv_attention(
32 self,
@@ -24,13 +35,11 @@ class OPTCacheFlowAttention(nn.Module):
35 key: torch.Tensor,
36 value: torch.Tensor,
37 ) -> None:
27 - query = query.unsqueeze(0)
28 - key = key.unsqueeze(0)
29 - value = value.unsqueeze(0)
30 - out = xops.memory_efficient_attention(
31 - query, key, value, attn_bias=self.attention_mask, scale=self.scale)
32 - out = out.squeeze(0)
33 - # FIXME(woosuk): Directly write the attention output.
38 + # FIXME(woosuk): Replace this with a custom op call.
39 + attention_mask = torch.triu(
40 + torch.ones(query.shape[0], key.shape[0]), diagonal=1) * -1e5
41 + attention_mask = attention_mask.to(dtype=query.dtype, device=query.device)
42 + out = self._masked_attention(query, key, value, attention_mask)
43 output.copy_(out, non_blocking=True)
44
45 def single_query_cached_kv_attention(
@@ -64,15 +73,10 @@ class OPTCacheFlowAttention(nn.Module):
73
74 v = value_cache[block_number, :, block_offset, :]
75 values.append(v)
67 -
76 keys = torch.stack(keys, dim=0)
77 values = torch.stack(values, dim=0)
78
71 - q = q.unsqueeze(0)
72 - keys = keys.unsqueeze(0)
73 - values = values.unsqueeze(0)
74 - out = xops.memory_efficient_attention(
75 - q, keys, values, scale=self.scale)
79 + out = self._masked_attention(q, keys, values)
80 out = out.view(num_heads, head_size)
81 output[i].copy_(out, non_blocking=True)
82