@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a31a817a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix FP8 KV cache support (#4869)

Woosuk Kwon committed May 16, 2024 at 15:42 UTC 9a31a817a85ac4249bf82dd8b6f90ef6b8e81fef
6 files changed +26 -26
vllm/attention/backends/flash_attn.py
+5 -5
@@ -200,15 +200,15 @@ class FlashAttentionImpl(AttentionImpl):
200 num_heads: int,
201 head_size: int,
202 scale: float,
203 - num_kv_heads: Optional[int] = None,
204 - alibi_slopes: Optional[List[float]] = None,
205 - sliding_window: Optional[int] = None,
206 - kv_cache_dtype: str = "auto",
203 + num_kv_heads: int,
204 + alibi_slopes: Optional[List[float]],
205 + sliding_window: Optional[int],
206 + kv_cache_dtype: str,
207 ) -> None:
208 self.num_heads = num_heads
209 self.head_size = head_size
210 self.scale = float(scale)
211 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
211 + self.num_kv_heads = num_kv_heads
212 if alibi_slopes is not None:
213 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
214 self.alibi_slopes = alibi_slopes
vllm/attention/backends/flashinfer.py
+5 -5
@@ -164,15 +164,15 @@ class FlashInferImpl(AttentionImpl):
164 num_heads: int,
165 head_size: int,
166 scale: float,
167 - num_kv_heads: Optional[int] = None,
168 - alibi_slopes: Optional[List[float]] = None,
169 - sliding_window: Optional[int] = None,
170 - kv_cache_dtype: str = "auto",
167 + num_kv_heads: int,
168 + alibi_slopes: Optional[List[float]],
169 + sliding_window: Optional[int],
170 + kv_cache_dtype: str,
171 ) -> None:
172 self.num_heads = num_heads
173 self.head_size = head_size
174 self.scale = float(scale)
175 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
175 + self.num_kv_heads = num_kv_heads
176 if alibi_slopes is not None:
177 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
178 self.alibi_slopes = alibi_slopes
vllm/attention/backends/rocm_flash_attn.py
+5 -5
@@ -197,15 +197,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
197 num_heads: int,
198 head_size: int,
199 scale: float,
200 - num_kv_heads: Optional[int] = None,
201 - alibi_slopes: Optional[List[float]] = None,
202 - sliding_window: Optional[int] = None,
203 - kv_cache_dtype: str = "auto",
200 + num_kv_heads: int,
201 + alibi_slopes: Optional[List[float]],
202 + sliding_window: Optional[int],
203 + kv_cache_dtype: str,
204 ) -> None:
205 self.num_heads = num_heads
206 self.head_size = head_size
207 self.scale = float(scale)
208 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208 + self.num_kv_heads = num_kv_heads
209 if alibi_slopes is not None:
210 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
211 self.alibi_slopes = alibi_slopes
vllm/attention/backends/torch_sdpa.py
+5 -5
@@ -96,15 +96,15 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
96 num_heads: int,
97 head_size: int,
98 scale: float,
99 - num_kv_heads: Optional[int] = None,
100 - alibi_slopes: Optional[List[float]] = None,
101 - sliding_window: Optional[int] = None,
102 - kv_cache_dtype: str = "auto",
99 + num_kv_heads: int,
100 + alibi_slopes: Optional[List[float]],
101 + sliding_window: Optional[int],
102 + kv_cache_dtype: str,
103 ) -> None:
104 self.num_heads = num_heads
105 self.head_size = head_size
106 self.scale = float(scale)
107 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
107 + self.num_kv_heads = num_kv_heads
108 if alibi_slopes is not None:
109 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
110 self.alibi_slopes = alibi_slopes
vllm/attention/backends/xformers.py
+5 -5
@@ -208,15 +208,15 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
208 num_heads: int,
209 head_size: int,
210 scale: float,
211 - num_kv_heads: Optional[int] = None,
212 - alibi_slopes: Optional[List[float]] = None,
213 - sliding_window: Optional[int] = None,
214 - kv_cache_dtype: str = "auto",
211 + num_kv_heads: int,
212 + alibi_slopes: Optional[List[float]],
213 + sliding_window: Optional[int],
214 + kv_cache_dtype: str,
215 ) -> None:
216 self.num_heads = num_heads
217 self.head_size = head_size
218 self.scale = float(scale)
219 - self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
219 + self.num_kv_heads = num_kv_heads
220 if alibi_slopes is not None:
221 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
222 self.alibi_slopes = alibi_slopes
vllm/attention/layer.py
+1 -1
@@ -48,7 +48,7 @@ class Attention(nn.Module):
48 block_size)
49 impl_cls = attn_backend.get_impl_cls()
50 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
51 - alibi_slopes, sliding_window)
51 + alibi_slopes, sliding_window, kv_cache_dtype)
52
53 def forward(
54 self,