A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Fix FP8 KV cache support (#4869)
Woosuk Kwon committed
May 16, 2024 at 15:42 UTC
9a31a817a85ac4249bf82dd8b6f90ef6b8e81fef
6 files changed
+26
-26
vllm/attention/backends/flash_attn.py
+5
-5
@@ -200,15 +200,15 @@ class FlashAttentionImpl(AttentionImpl):
200
num_heads: int,
201
head_size: int,
202
scale: float,
203
- num_kv_heads: Optional[int] = None,
204
- alibi_slopes: Optional[List[float]] = None,
205
- sliding_window: Optional[int] = None,
206
- kv_cache_dtype: str = "auto",
203
+ num_kv_heads: int,
204
+ alibi_slopes: Optional[List[float]],
205
+ sliding_window: Optional[int],
206
+ kv_cache_dtype: str,
207
) -> None:
208
self.num_heads = num_heads
209
self.head_size = head_size
210
self.scale = float(scale)
211
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
211
+ self.num_kv_heads = num_kv_heads
212
if alibi_slopes is not None:
213
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
214
self.alibi_slopes = alibi_slopes
vllm/attention/backends/flashinfer.py
+5
-5
@@ -164,15 +164,15 @@ class FlashInferImpl(AttentionImpl):
164
num_heads: int,
165
head_size: int,
166
scale: float,
167
- num_kv_heads: Optional[int] = None,
168
- alibi_slopes: Optional[List[float]] = None,
169
- sliding_window: Optional[int] = None,
170
- kv_cache_dtype: str = "auto",
167
+ num_kv_heads: int,
168
+ alibi_slopes: Optional[List[float]],
169
+ sliding_window: Optional[int],
170
+ kv_cache_dtype: str,
171
) -> None:
172
self.num_heads = num_heads
173
self.head_size = head_size
174
self.scale = float(scale)
175
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
175
+ self.num_kv_heads = num_kv_heads
176
if alibi_slopes is not None:
177
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
178
self.alibi_slopes = alibi_slopes
vllm/attention/backends/rocm_flash_attn.py
+5
-5
@@ -197,15 +197,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
197
num_heads: int,
198
head_size: int,
199
scale: float,
200
- num_kv_heads: Optional[int] = None,
201
- alibi_slopes: Optional[List[float]] = None,
202
- sliding_window: Optional[int] = None,
203
- kv_cache_dtype: str = "auto",
200
+ num_kv_heads: int,
201
+ alibi_slopes: Optional[List[float]],
202
+ sliding_window: Optional[int],
203
+ kv_cache_dtype: str,
204
) -> None:
205
self.num_heads = num_heads
206
self.head_size = head_size
207
self.scale = float(scale)
208
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208
+ self.num_kv_heads = num_kv_heads
209
if alibi_slopes is not None:
210
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
211
self.alibi_slopes = alibi_slopes
vllm/attention/backends/torch_sdpa.py
+5
-5
@@ -96,15 +96,15 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
96
num_heads: int,
97
head_size: int,
98
scale: float,
99
- num_kv_heads: Optional[int] = None,
100
- alibi_slopes: Optional[List[float]] = None,
101
- sliding_window: Optional[int] = None,
102
- kv_cache_dtype: str = "auto",
99
+ num_kv_heads: int,
100
+ alibi_slopes: Optional[List[float]],
101
+ sliding_window: Optional[int],
102
+ kv_cache_dtype: str,
103
) -> None:
104
self.num_heads = num_heads
105
self.head_size = head_size
106
self.scale = float(scale)
107
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
107
+ self.num_kv_heads = num_kv_heads
108
if alibi_slopes is not None:
109
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
110
self.alibi_slopes = alibi_slopes
vllm/attention/backends/xformers.py
+5
-5
@@ -208,15 +208,15 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
208
num_heads: int,
209
head_size: int,
210
scale: float,
211
- num_kv_heads: Optional[int] = None,
212
- alibi_slopes: Optional[List[float]] = None,
213
- sliding_window: Optional[int] = None,
214
- kv_cache_dtype: str = "auto",
211
+ num_kv_heads: int,
212
+ alibi_slopes: Optional[List[float]],
213
+ sliding_window: Optional[int],
214
+ kv_cache_dtype: str,
215
) -> None:
216
self.num_heads = num_heads
217
self.head_size = head_size
218
self.scale = float(scale)
219
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
219
+ self.num_kv_heads = num_kv_heads
220
if alibi_slopes is not None:
221
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
222
self.alibi_slopes = alibi_slopes
vllm/attention/layer.py
+1
-1
@@ -48,7 +48,7 @@ class Attention(nn.Module):
48
block_size)
49
impl_cls = attn_backend.get_impl_cls()
50
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
51
- alibi_slopes, sliding_window)
51
+ alibi_slopes, sliding_window, kv_cache_dtype)
52
53
def forward(
54
self,