A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core/Bugfix] Add FP8 K/V Scale and dtype conversion for prefix/prefill Triton Kernel (#7208)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
jon-chuang committed
Aug 12, 2024 at 15:47 UTC
a046f86397c06af306d964ff40d0670bdc9c00a2
10 files changed
+208
-47
docs/source/quantization/fp8_e4m3_kvcache.rst
-2
@@ -45,5 +45,3 @@ Here is an example of how to enable this feature:
45
# output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
46
# output w/o scaling factors: England, located in the southeastern part of the country. It is known
47
48
-Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
49
-
docs/source/quantization/fp8_e5m2_kvcache.rst
-2
@@ -32,5 +32,3 @@ Here is an example of how to enable this feature:
32
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33
34
35
-Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
36
-
tests/basic_correctness/test_chunked_prefill.py
+96
-8
@@ -6,14 +6,27 @@ prefill requests are chunked.
6
7
Run `pytest tests/models/test_chunked_prefill.py`.
8
"""
9
+
10
import pytest
11
11
-from ..models.utils import check_outputs_equal
12
+from ..models.utils import check_logprobs_close, check_outputs_equal
13
14
MODELS = [
15
"facebook/opt-125m",
16
"meta-llama/Llama-2-7b-hf",
17
]
18
+E5M2_KV_MODELS = [
19
+ "facebook/opt-125m",
20
+ "meta-llama/Llama-2-7b-chat-hf",
21
+]
22
+E4M3_KV_MODELS = [
23
+ "meta-llama/Llama-2-7b-chat-hf", "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
24
+ "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
25
+]
26
+KV_CACHE_QUANTIZATION_PATHS = {
27
+ "meta-llama/Llama-2-7b-chat-hf":
28
+ "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json"
29
+}
30
31
32
@pytest.mark.parametrize("model", MODELS)
@@ -35,12 +48,12 @@ def test_models(
48
enforce_eager: bool,
49
tensor_parallel_size: int,
50
) -> None:
38
- max_num_seqs = min(chunked_prefill_token_size, 256)
39
- enable_chunked_prefill = False
40
- max_num_batched_tokens = None
41
- if chunked_prefill_token_size != -1:
42
- enable_chunked_prefill = True
43
- max_num_batched_tokens = chunked_prefill_token_size
51
+ """
52
+ Checks exact match decode between huggingface model and vllm runner with
53
+ chunked prefill.
54
+ """
55
+ max_num_seqs = chunked_prefill_token_size
56
+ max_num_batched_tokens = chunked_prefill_token_size
57
58
with hf_runner(model, dtype=dtype) as hf_model:
59
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
@@ -49,7 +62,7 @@ def test_models(
62
model,
63
dtype=dtype,
64
max_num_batched_tokens=max_num_batched_tokens,
52
- enable_chunked_prefill=enable_chunked_prefill,
65
+ enable_chunked_prefill=True,
66
tensor_parallel_size=tensor_parallel_size,
67
enforce_eager=enforce_eager,
68
max_num_seqs=max_num_seqs,
@@ -62,3 +75,78 @@ def test_models(
75
name_0="hf",
76
name_1="vllm",
77
)
78
+
79
+
80
+@pytest.mark.parametrize("kv_cache_dtype,model",
81
+ [("fp8_e5m2", m)
82
+ for m in E5M2_KV_MODELS] + [("fp8_e4m3", m)
83
+ for m in E4M3_KV_MODELS])
84
+# Due to low-precision numerical divergence, we only test logprob of 4 tokens
85
+@pytest.mark.parametrize("max_tokens", [4])
86
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
87
+@pytest.mark.parametrize("enforce_eager", [False, True])
88
+# NOTE: Increasing this in this suite will fail CI because we currently cannot
89
+# reset distributed env properly. Use a value > 1 just when you test.
90
+@pytest.mark.parametrize("tensor_parallel_size", [1])
91
+def test_models_with_fp8_kv_cache(
92
+ vllm_runner,
93
+ example_prompts,
94
+ kv_cache_dtype: str,
95
+ model: str,
96
+ max_tokens: int,
97
+ chunked_prefill_token_size: int,
98
+ enforce_eager: bool,
99
+ tensor_parallel_size: int,
100
+) -> None:
101
+ """
102
+ Only checks log probs match between chunked-prefill and
103
+ non-chunked-prefill version of vLLM model runner.
104
+
105
+ This test is used when there is discrepancy in kernels
106
+ / numerics (e.g. when using lower-precision types like FP8).
107
+ """
108
+ NUM_LOG_PROBS = 8
109
+
110
+ if model == "facebook/opt-125m":
111
+ pytest.skip(
112
+ "#7378: CUDA illegal memory access (undiagnosed) facebook/opt-125m"
113
+ )
114
+
115
+ max_num_seqs = chunked_prefill_token_size
116
+ max_num_batched_tokens = chunked_prefill_token_size
117
+
118
+ extra_kwargs = {}
119
+ if model in KV_CACHE_QUANTIZATION_PATHS:
120
+ extra_kwargs["quantization_param_path"] = KV_CACHE_QUANTIZATION_PATHS[
121
+ model]
122
+
123
+ with vllm_runner(
124
+ model,
125
+ tensor_parallel_size=tensor_parallel_size,
126
+ enforce_eager=enforce_eager,
127
+ max_num_seqs=max_num_seqs,
128
+ kv_cache_dtype=kv_cache_dtype,
129
+ **extra_kwargs,
130
+ ) as vllm_model:
131
+ no_chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
132
+ example_prompts, max_tokens, NUM_LOG_PROBS)
133
+
134
+ with vllm_runner(
135
+ model,
136
+ max_num_batched_tokens=max_num_batched_tokens,
137
+ enable_chunked_prefill=True,
138
+ tensor_parallel_size=tensor_parallel_size,
139
+ enforce_eager=enforce_eager,
140
+ max_num_seqs=max_num_seqs,
141
+ kv_cache_dtype=kv_cache_dtype,
142
+ **extra_kwargs,
143
+ ) as vllm_model:
144
+ chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
145
+ example_prompts, max_tokens, NUM_LOG_PROBS)
146
+
147
+ check_logprobs_close(
148
+ outputs_0_lst=no_chunked_prefill_outputs,
149
+ outputs_1_lst=chunked_prefill_outputs,
150
+ name_0="no_chunked_prefill",
151
+ name_1="chunked_prefill",
152
+ )
tests/kernels/test_prefix_prefill.py
+26
-7
@@ -9,6 +9,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
10
from vllm.attention.backends.xformers import _make_alibi_bias
11
from vllm.attention.ops.prefix_prefill import context_attention_fwd
12
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
13
14
NUM_HEADS = [64]
15
NUM_QUERIES_PER_KV = [1, 8, 64]
@@ -18,12 +19,14 @@ CUDA_DEVICES = [
19
f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
20
]
21
SLIDING_WINDOW = [0, 16, 64, 128, 256, 512, 2048]
22
+KV_CACHE_DTYPES = ["auto", "fp8", "fp8_e5m2"]
23
24
25
@pytest.mark.parametrize("num_heads", NUM_HEADS)
26
@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
27
@pytest.mark.parametrize("head_size", HEAD_SIZES)
28
@pytest.mark.parametrize("dtype", DTYPES)
29
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
30
@pytest.mark.parametrize("device", CUDA_DEVICES)
31
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
32
@torch.inference_mode()
@@ -33,6 +36,7 @@ def test_contexted_kv_attention(
36
head_size: int,
37
sliding_window: int,
38
dtype: torch.dtype,
39
+ kv_cache_dtype: str,
40
device: str,
41
) -> None:
42
random.seed(0)
@@ -67,16 +71,20 @@ def test_contexted_kv_attention(
71
kv.uniform_(-1e-3, 1e-3)
72
key, value = kv.unbind(dim=1)
73
74
+ if kv_cache_dtype == "auto":
75
+ cache_dtype = dtype
76
+ else:
77
+ cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[kv_cache_dtype]
78
k_cache = torch.zeros(cache_size,
79
block_size,
80
num_kv_heads,
81
head_size,
74
- dtype=dtype)
82
+ dtype=cache_dtype)
83
v_cache = torch.zeros(cache_size,
84
block_size,
85
num_kv_heads,
86
head_size,
79
- dtype=dtype)
87
+ dtype=cache_dtype)
88
k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
89
v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
90
values = torch.arange(0, cache_size, dtype=torch.long)
@@ -132,6 +140,7 @@ def test_contexted_kv_attention(
140
k,
141
v,
142
output,
143
+ kv_cache_dtype,
144
k_cache,
145
v_cache,
146
block_table,
@@ -146,6 +155,7 @@ def test_contexted_kv_attention(
155
k,
156
v,
157
output,
158
+ kv_cache_dtype,
159
k_cache,
160
v_cache,
161
block_table,
@@ -208,13 +218,15 @@ def test_contexted_kv_attention(
218
end_time = time.time()
219
print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
220
output_ref = output_ref.reshape(output.shape)
211
- assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
221
+ atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
222
+ torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
223
224
225
@pytest.mark.parametrize("num_heads", NUM_HEADS)
226
@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
227
@pytest.mark.parametrize("head_size", HEAD_SIZES)
228
@pytest.mark.parametrize("dtype", DTYPES)
229
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
230
@pytest.mark.parametrize("device", CUDA_DEVICES)
231
@torch.inference_mode()
232
def test_contexted_kv_attention_alibi(
@@ -222,6 +234,7 @@ def test_contexted_kv_attention_alibi(
234
num_queries_per_kv: int,
235
head_size: int,
236
dtype: torch.dtype,
237
+ kv_cache_dtype: str,
238
device: str,
239
) -> None:
240
random.seed(0)
@@ -282,17 +295,20 @@ def test_contexted_kv_attention_alibi(
295
kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
296
kv.uniform_(-1e-3, 1e-3)
297
key, value = kv.unbind(dim=1)
285
-
298
+ if kv_cache_dtype == "auto":
299
+ cache_dtype = dtype
300
+ else:
301
+ cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[kv_cache_dtype]
302
k_cache = torch.zeros(cache_size,
303
block_size,
304
num_kv_heads,
305
head_size,
290
- dtype=dtype)
306
+ dtype=cache_dtype)
307
v_cache = torch.zeros(cache_size,
308
block_size,
309
num_kv_heads,
310
head_size,
295
- dtype=dtype)
311
+ dtype=cache_dtype)
312
k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
313
v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
314
values = torch.arange(0, cache_size, dtype=torch.long)
@@ -348,6 +364,7 @@ def test_contexted_kv_attention_alibi(
364
k,
365
v,
366
output,
367
+ kv_cache_dtype,
368
k_cache,
369
v_cache,
370
block_table,
@@ -362,6 +379,7 @@ def test_contexted_kv_attention_alibi(
379
k,
380
v,
381
output,
382
+ kv_cache_dtype,
383
k_cache,
384
v_cache,
385
block_table,
@@ -447,4 +465,5 @@ def test_contexted_kv_attention_alibi(
465
torch.cuda.synchronize()
466
end_time = time.time()
467
print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
450
- assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
468
+ atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
469
+ torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
vllm/attention/backends/rocm_flash_attn.py
+3
@@ -459,6 +459,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
459
query,
460
key,
461
value,
462
+ self.kv_cache_dtype,
463
key_cache,
464
value_cache,
465
prefill_meta.block_tables,
@@ -468,6 +469,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
469
prefill_meta.max_query_len,
470
self.alibi_slopes,
471
self.sliding_window[0],
472
+ k_scale,
473
+ v_scale,
474
)
475
476
if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/xformers.py
+3
@@ -604,6 +604,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
604
query,
605
key,
606
value,
607
+ self.kv_cache_dtype,
608
key_cache,
609
value_cache,
610
prefill_meta.block_tables,
@@ -613,6 +614,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
614
prefill_meta.max_query_len,
615
self.alibi_slopes,
616
self.sliding_window,
617
+ k_scale,
618
+ v_scale,
619
)
620
assert output[:num_prefill_tokens].shape == out.shape
621
output[:num_prefill_tokens] = out
vllm/attention/ops/ipex_attn.py
+1
@@ -90,6 +90,7 @@ class PagedAttention:
90
query: torch.Tensor,
91
key: torch.Tensor,
92
value: torch.Tensor,
93
+ kv_cache_dtype: str,
94
key_cache: torch.Tensor,
95
value_cache: torch.Tensor,
96
block_tables: torch.Tensor,
vllm/attention/ops/paged_attn.py
+6
@@ -194,6 +194,7 @@ class PagedAttention:
194
query: torch.Tensor,
195
key: torch.Tensor,
196
value: torch.Tensor,
197
+ kv_cache_dtype: str,
198
key_cache: torch.Tensor,
199
value_cache: torch.Tensor,
200
block_tables: torch.Tensor,
@@ -203,6 +204,8 @@ class PagedAttention:
204
max_query_len: int,
205
alibi_slopes: Optional[torch.Tensor],
206
sliding_window: Optional[int],
207
+ k_scale: float,
208
+ v_scale: float,
209
) -> torch.Tensor:
210
output = torch.empty_like(query)
211
context_attention_fwd(
@@ -210,6 +213,7 @@ class PagedAttention:
213
key,
214
value,
215
output,
216
+ kv_cache_dtype,
217
key_cache,
218
value_cache,
219
block_tables,
@@ -218,6 +222,8 @@ class PagedAttention:
222
seq_lens_tensor,
223
context_lens,
224
max_query_len,
225
+ k_scale,
226
+ v_scale,
227
alibi_slopes,
228
sliding_window,
229
)
vllm/attention/ops/prefix_prefill.py
+73
-24
@@ -18,6 +18,8 @@ if triton.__version__ >= "2.1.0":
18
V_cache,
19
B_Loc,
20
sm_scale,
21
+ k_scale,
22
+ v_scale,
23
B_Start_Loc,
24
B_Seqlen,
25
B_Ctxlen,
@@ -117,10 +119,15 @@ if triton.__version__ >= "2.1.0":
119
cur_kv_head * stride_v_cache_h +
120
offs_d[None, :] * stride_v_cache_d +
121
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
120
- k = tl.load(K_cache + off_k,
121
- mask=dim_mask[:, None] &
122
- ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
123
- other=0.0) # [D,N]
122
+ k_load = tl.load(K_cache + off_k,
123
+ mask=dim_mask[:, None] &
124
+ ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
125
+ other=0.0) # [D,N]
126
+
127
+ if k_load.dtype.is_fp8():
128
+ k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
129
+ else:
130
+ k = k_load
131
132
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
133
qk += tl.dot(q, k)
@@ -161,12 +168,16 @@ if triton.__version__ >= "2.1.0":
168
acc_scale = l_i / l_i_new * alpha
169
acc = acc * acc_scale[:, None]
170
# update acc
164
- v = tl.load(V_cache + off_v,
165
- mask=dim_mask[None, :] &
166
- ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
167
- other=0.0) # [N,D]
168
-
171
+ v_load = tl.load(V_cache + off_v,
172
+ mask=dim_mask[None, :] &
173
+ ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
174
+ other=0.0) # [N,D]
175
+ if v_load.dtype.is_fp8():
176
+ v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
177
+ else:
178
+ v = v_load
179
p = p.to(v.dtype)
180
+
181
acc += tl.dot(p, v)
182
# # update m_i and l_i
183
l_i = l_i_new
@@ -225,8 +236,8 @@ if triton.__version__ >= "2.1.0":
236
mask=dim_mask[None, :] &
237
((start_n + offs_n[:, None]) < cur_batch_query_len),
238
other=0.0)
228
-
239
p = p.to(v.dtype)
240
+
241
acc += tl.dot(p, v)
242
# update m_i and l_i
243
l_i = l_i_new
@@ -336,7 +347,6 @@ if triton.__version__ >= "2.1.0":
347
k = tl.load(K_cache + off_k,
348
mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
349
other=0.0)
339
-
350
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
351
qk += tl.dot(q, k)
352
qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
@@ -442,6 +452,8 @@ if triton.__version__ >= "2.1.0":
452
V_cache,
453
B_Loc,
454
sm_scale,
455
+ k_scale,
456
+ v_scale,
457
B_Start_Loc,
458
B_Seqlen,
459
B_Ctxlen,
@@ -537,10 +549,15 @@ if triton.__version__ >= "2.1.0":
549
cur_kv_head * stride_v_cache_h +
550
offs_d[None, :] * stride_v_cache_d +
551
(start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
540
- k = tl.load(K_cache + off_k,
541
- mask=dim_mask[:, None] &
542
- ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
543
- other=0.0) # [D,N]
552
+ k_load = tl.load(K_cache + off_k,
553
+ mask=dim_mask[:, None] &
554
+ ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
555
+ other=0.0) # [D,N]
556
+
557
+ if k_load.dtype.is_fp8():
558
+ k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
559
+ else:
560
+ k = k_load
561
562
qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
563
qk += tl.dot(q, k)
@@ -573,12 +590,16 @@ if triton.__version__ >= "2.1.0":
590
# acc_scale = l_i / l_i_new * alpha
591
acc = acc * acc_scale[:, None]
592
# update acc
576
- v = tl.load(V_cache + off_v,
577
- mask=dim_mask[None, :] &
578
- ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
579
- other=0.0)
580
-
593
+ v_load = tl.load(V_cache + off_v,
594
+ mask=dim_mask[None, :] &
595
+ ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
596
+ other=0.0)
597
+ if v_load.dtype.is_fp8():
598
+ v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
599
+ else:
600
+ v = v_load
601
p = p.to(v.dtype)
602
+
603
acc += tl.dot(p, v, allow_tf32=False)
604
# update m_i and l_i
605
l_i = l_i_new
@@ -650,8 +671,8 @@ if triton.__version__ >= "2.1.0":
671
((start_n + offs_n[:, None]) <
672
cur_batch_seq_len - cur_batch_ctx_len),
673
other=0.0)
653
-
674
p = p.to(v.dtype)
675
+
676
acc += tl.dot(p, v, allow_tf32=False)
677
# update m_i and l_i
678
l_i = l_i_new
@@ -675,6 +696,7 @@ if triton.__version__ >= "2.1.0":
696
k,
697
v,
698
o,
699
+ kv_cache_dtype: str,
700
k_cache,
701
v_cache,
702
b_loc,
@@ -682,17 +704,41 @@ if triton.__version__ >= "2.1.0":
704
b_seq_len,
705
b_ctx_len,
706
max_input_len,
707
+ k_scale: float = 1.0,
708
+ v_scale: float = 1.0,
709
alibi_slopes=None,
710
sliding_window=None):
711
712
cap = current_platform.get_device_capability()
713
BLOCK = 128 if cap[0] >= 8 else 64
714
+ NUM_WARPS = 8
715
716
# need to reduce num. blocks when using fp32
717
# due to increased use of GPU shared memory
718
if q.dtype is torch.float32:
719
BLOCK = BLOCK // 2
720
721
+ # Conversion of FP8 Tensor from uint8 storage to
722
+ # appropriate torch.dtype for interpretation by Triton
723
+ if "fp8" in kv_cache_dtype:
724
+ assert (k_cache.dtype == torch.uint8)
725
+ assert (v_cache.dtype == torch.uint8)
726
+
727
+ if kv_cache_dtype in ("fp8", "fp8_e4m3"):
728
+ target_dtype = torch.float8_e4m3fn
729
+ elif kv_cache_dtype == "fp8_e5m2":
730
+ target_dtype = torch.float8_e5m2
731
+ else:
732
+ raise ValueError("Unsupported FP8 dtype:", kv_cache_dtype)
733
+
734
+ k_cache = k_cache.view(target_dtype)
735
+ v_cache = v_cache.view(target_dtype)
736
+
737
+ if (k_cache.dtype == torch.uint8
738
+ or v_cache.dtype == torch.uint8 and kv_cache_dtype == "auto"):
739
+ raise ValueError("kv_cache_dtype='auto' unsupported for\
740
+ FP8 KV Cache prefill kernel")
741
+
742
# shape constraints
743
Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
744
assert Lq == Lk and Lk == Lv
@@ -709,7 +755,6 @@ if triton.__version__ >= "2.1.0":
755
if sliding_window is None or sliding_window <= 0:
756
sliding_window = 0
757
712
- num_warps = 8 if Lk <= 64 else 8
758
if alibi_slopes is not None:
759
_fwd_kernel_alibi[grid](
760
q,
@@ -719,6 +764,8 @@ if triton.__version__ >= "2.1.0":
764
v_cache,
765
b_loc,
766
sm_scale,
767
+ k_scale,
768
+ v_scale,
769
b_start_loc,
770
b_seq_len,
771
b_ctx_len,
@@ -757,7 +804,7 @@ if triton.__version__ >= "2.1.0":
804
BLOCK_DMODEL=Lk,
805
BLOCK_DMODEL_PADDED=Lk_padded,
806
BLOCK_N=BLOCK,
760
- num_warps=num_warps,
807
+ num_warps=NUM_WARPS,
808
num_stages=1,
809
)
810
return
@@ -770,6 +817,8 @@ if triton.__version__ >= "2.1.0":
817
v_cache,
818
b_loc,
819
sm_scale,
820
+ k_scale,
821
+ v_scale,
822
b_start_loc,
823
b_seq_len,
824
b_ctx_len,
@@ -807,7 +856,7 @@ if triton.__version__ >= "2.1.0":
856
BLOCK_DMODEL_PADDED=Lk_padded,
857
BLOCK_N=BLOCK,
858
SLIDING_WINDOW=sliding_window,
810
- num_warps=num_warps,
859
+ num_warps=NUM_WARPS,
860
num_stages=1,
861
)
862
return
vllm/config.py
-4
@@ -545,10 +545,6 @@ class CacheConfig:
545
raise NotImplementedError(
546
"Prefix caching is not supported with sliding window. "
547
"Run with --disable-sliding-window to use prefix caching.")
548
- if self.cache_dtype == "fp8":
549
- raise NotImplementedError(
550
- "Prefix caching is not supported for fp8 cache_dtype. "
551
- "Run with --kv-cache-dtype auto to use prefix caching.")
548
549
def verify_with_parallel_config(
550
self,