@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a046f86397
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core/Bugfix] Add FP8 K/V Scale and dtype conversion for prefix/prefill Triton Kernel (#7208)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

jon-chuang committed Aug 12, 2024 at 15:47 UTC a046f86397c06af306d964ff40d0670bdc9c00a2
10 files changed +208 -47
docs/source/quantization/fp8_e4m3_kvcache.rst
-2
@@ -45,5 +45,3 @@ Here is an example of how to enable this feature:
45 # output w/ scaling factors: England, the United Kingdom, and one of the world's leading financial,
46 # output w/o scaling factors: England, located in the southeastern part of the country. It is known
47
48 -Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
49 -
docs/source/quantization/fp8_e5m2_kvcache.rst
-2
@@ -32,5 +32,3 @@ Here is an example of how to enable this feature:
32 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
33
34
35 -Note, current prefix caching doesn't work with FP8 KV cache enabled, forward_prefix kernel should handle different KV and cache type.
36 -
tests/basic_correctness/test_chunked_prefill.py
+96 -8
@@ -6,14 +6,27 @@ prefill requests are chunked.
6
7 Run `pytest tests/models/test_chunked_prefill.py`.
8 """
9 +
10 import pytest
11
11 -from ..models.utils import check_outputs_equal
12 +from ..models.utils import check_logprobs_close, check_outputs_equal
13
14 MODELS = [
15 "facebook/opt-125m",
16 "meta-llama/Llama-2-7b-hf",
17 ]
18 +E5M2_KV_MODELS = [
19 + "facebook/opt-125m",
20 + "meta-llama/Llama-2-7b-chat-hf",
21 +]
22 +E4M3_KV_MODELS = [
23 + "meta-llama/Llama-2-7b-chat-hf", "nm-testing/Qwen2-1.5B-Instruct-FP8-K-V",
24 + "nm-testing/TinyLlama-1.1B-compressed-tensors-kv-cache-scheme"
25 +]
26 +KV_CACHE_QUANTIZATION_PATHS = {
27 + "meta-llama/Llama-2-7b-chat-hf":
28 + "./tests/fp8_kv/llama2-7b-fp8-kv/kv_cache_scales.json"
29 +}
30
31
32 @pytest.mark.parametrize("model", MODELS)
@@ -35,12 +48,12 @@ def test_models(
48 enforce_eager: bool,
49 tensor_parallel_size: int,
50 ) -> None:
38 - max_num_seqs = min(chunked_prefill_token_size, 256)
39 - enable_chunked_prefill = False
40 - max_num_batched_tokens = None
41 - if chunked_prefill_token_size != -1:
42 - enable_chunked_prefill = True
43 - max_num_batched_tokens = chunked_prefill_token_size
51 + """
52 + Checks exact match decode between huggingface model and vllm runner with
53 + chunked prefill.
54 + """
55 + max_num_seqs = chunked_prefill_token_size
56 + max_num_batched_tokens = chunked_prefill_token_size
57
58 with hf_runner(model, dtype=dtype) as hf_model:
59 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
@@ -49,7 +62,7 @@ def test_models(
62 model,
63 dtype=dtype,
64 max_num_batched_tokens=max_num_batched_tokens,
52 - enable_chunked_prefill=enable_chunked_prefill,
65 + enable_chunked_prefill=True,
66 tensor_parallel_size=tensor_parallel_size,
67 enforce_eager=enforce_eager,
68 max_num_seqs=max_num_seqs,
@@ -62,3 +75,78 @@ def test_models(
75 name_0="hf",
76 name_1="vllm",
77 )
78 +
79 +
80 +@pytest.mark.parametrize("kv_cache_dtype,model",
81 + [("fp8_e5m2", m)
82 + for m in E5M2_KV_MODELS] + [("fp8_e4m3", m)
83 + for m in E4M3_KV_MODELS])
84 +# Due to low-precision numerical divergence, we only test logprob of 4 tokens
85 +@pytest.mark.parametrize("max_tokens", [4])
86 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
87 +@pytest.mark.parametrize("enforce_eager", [False, True])
88 +# NOTE: Increasing this in this suite will fail CI because we currently cannot
89 +# reset distributed env properly. Use a value > 1 just when you test.
90 +@pytest.mark.parametrize("tensor_parallel_size", [1])
91 +def test_models_with_fp8_kv_cache(
92 + vllm_runner,
93 + example_prompts,
94 + kv_cache_dtype: str,
95 + model: str,
96 + max_tokens: int,
97 + chunked_prefill_token_size: int,
98 + enforce_eager: bool,
99 + tensor_parallel_size: int,
100 +) -> None:
101 + """
102 + Only checks log probs match between chunked-prefill and
103 + non-chunked-prefill version of vLLM model runner.
104 +
105 + This test is used when there is discrepancy in kernels
106 + / numerics (e.g. when using lower-precision types like FP8).
107 + """
108 + NUM_LOG_PROBS = 8
109 +
110 + if model == "facebook/opt-125m":
111 + pytest.skip(
112 + "#7378: CUDA illegal memory access (undiagnosed) facebook/opt-125m"
113 + )
114 +
115 + max_num_seqs = chunked_prefill_token_size
116 + max_num_batched_tokens = chunked_prefill_token_size
117 +
118 + extra_kwargs = {}
119 + if model in KV_CACHE_QUANTIZATION_PATHS:
120 + extra_kwargs["quantization_param_path"] = KV_CACHE_QUANTIZATION_PATHS[
121 + model]
122 +
123 + with vllm_runner(
124 + model,
125 + tensor_parallel_size=tensor_parallel_size,
126 + enforce_eager=enforce_eager,
127 + max_num_seqs=max_num_seqs,
128 + kv_cache_dtype=kv_cache_dtype,
129 + **extra_kwargs,
130 + ) as vllm_model:
131 + no_chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
132 + example_prompts, max_tokens, NUM_LOG_PROBS)
133 +
134 + with vllm_runner(
135 + model,
136 + max_num_batched_tokens=max_num_batched_tokens,
137 + enable_chunked_prefill=True,
138 + tensor_parallel_size=tensor_parallel_size,
139 + enforce_eager=enforce_eager,
140 + max_num_seqs=max_num_seqs,
141 + kv_cache_dtype=kv_cache_dtype,
142 + **extra_kwargs,
143 + ) as vllm_model:
144 + chunked_prefill_outputs = vllm_model.generate_greedy_logprobs(
145 + example_prompts, max_tokens, NUM_LOG_PROBS)
146 +
147 + check_logprobs_close(
148 + outputs_0_lst=no_chunked_prefill_outputs,
149 + outputs_1_lst=chunked_prefill_outputs,
150 + name_0="no_chunked_prefill",
151 + name_1="chunked_prefill",
152 + )
tests/kernels/test_prefix_prefill.py
+26 -7
@@ -9,6 +9,7 @@ from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
10 from vllm.attention.backends.xformers import _make_alibi_bias
11 from vllm.attention.ops.prefix_prefill import context_attention_fwd
12 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
13
14 NUM_HEADS = [64]
15 NUM_QUERIES_PER_KV = [1, 8, 64]
@@ -18,12 +19,14 @@ CUDA_DEVICES = [
19 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
20 ]
21 SLIDING_WINDOW = [0, 16, 64, 128, 256, 512, 2048]
22 +KV_CACHE_DTYPES = ["auto", "fp8", "fp8_e5m2"]
23
24
25 @pytest.mark.parametrize("num_heads", NUM_HEADS)
26 @pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
27 @pytest.mark.parametrize("head_size", HEAD_SIZES)
28 @pytest.mark.parametrize("dtype", DTYPES)
29 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
30 @pytest.mark.parametrize("device", CUDA_DEVICES)
31 @pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
32 @torch.inference_mode()
@@ -33,6 +36,7 @@ def test_contexted_kv_attention(
36 head_size: int,
37 sliding_window: int,
38 dtype: torch.dtype,
39 + kv_cache_dtype: str,
40 device: str,
41 ) -> None:
42 random.seed(0)
@@ -67,16 +71,20 @@ def test_contexted_kv_attention(
71 kv.uniform_(-1e-3, 1e-3)
72 key, value = kv.unbind(dim=1)
73
74 + if kv_cache_dtype == "auto":
75 + cache_dtype = dtype
76 + else:
77 + cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[kv_cache_dtype]
78 k_cache = torch.zeros(cache_size,
79 block_size,
80 num_kv_heads,
81 head_size,
74 - dtype=dtype)
82 + dtype=cache_dtype)
83 v_cache = torch.zeros(cache_size,
84 block_size,
85 num_kv_heads,
86 head_size,
79 - dtype=dtype)
87 + dtype=cache_dtype)
88 k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
89 v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
90 values = torch.arange(0, cache_size, dtype=torch.long)
@@ -132,6 +140,7 @@ def test_contexted_kv_attention(
140 k,
141 v,
142 output,
143 + kv_cache_dtype,
144 k_cache,
145 v_cache,
146 block_table,
@@ -146,6 +155,7 @@ def test_contexted_kv_attention(
155 k,
156 v,
157 output,
158 + kv_cache_dtype,
159 k_cache,
160 v_cache,
161 block_table,
@@ -208,13 +218,15 @@ def test_contexted_kv_attention(
218 end_time = time.time()
219 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
220 output_ref = output_ref.reshape(output.shape)
211 - assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
221 + atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
222 + torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
223
224
225 @pytest.mark.parametrize("num_heads", NUM_HEADS)
226 @pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
227 @pytest.mark.parametrize("head_size", HEAD_SIZES)
228 @pytest.mark.parametrize("dtype", DTYPES)
229 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
230 @pytest.mark.parametrize("device", CUDA_DEVICES)
231 @torch.inference_mode()
232 def test_contexted_kv_attention_alibi(
@@ -222,6 +234,7 @@ def test_contexted_kv_attention_alibi(
234 num_queries_per_kv: int,
235 head_size: int,
236 dtype: torch.dtype,
237 + kv_cache_dtype: str,
238 device: str,
239 ) -> None:
240 random.seed(0)
@@ -282,17 +295,20 @@ def test_contexted_kv_attention_alibi(
295 kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
296 kv.uniform_(-1e-3, 1e-3)
297 key, value = kv.unbind(dim=1)
285 -
298 + if kv_cache_dtype == "auto":
299 + cache_dtype = dtype
300 + else:
301 + cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[kv_cache_dtype]
302 k_cache = torch.zeros(cache_size,
303 block_size,
304 num_kv_heads,
305 head_size,
290 - dtype=dtype)
306 + dtype=cache_dtype)
307 v_cache = torch.zeros(cache_size,
308 block_size,
309 num_kv_heads,
310 head_size,
295 - dtype=dtype)
311 + dtype=cache_dtype)
312 k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
313 v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
314 values = torch.arange(0, cache_size, dtype=torch.long)
@@ -348,6 +364,7 @@ def test_contexted_kv_attention_alibi(
364 k,
365 v,
366 output,
367 + kv_cache_dtype,
368 k_cache,
369 v_cache,
370 block_table,
@@ -362,6 +379,7 @@ def test_contexted_kv_attention_alibi(
379 k,
380 v,
381 output,
382 + kv_cache_dtype,
383 k_cache,
384 v_cache,
385 block_table,
@@ -447,4 +465,5 @@ def test_contexted_kv_attention_alibi(
465 torch.cuda.synchronize()
466 end_time = time.time()
467 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
450 - assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
468 + atol = 1e-3 if "fp8" in kv_cache_dtype else 1e-6
469 + torch.testing.assert_close(output, output_ref, atol=atol, rtol=0)
vllm/attention/backends/rocm_flash_attn.py
+3
@@ -459,6 +459,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
459 query,
460 key,
461 value,
462 + self.kv_cache_dtype,
463 key_cache,
464 value_cache,
465 prefill_meta.block_tables,
@@ -468,6 +469,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
469 prefill_meta.max_query_len,
470 self.alibi_slopes,
471 self.sliding_window[0],
472 + k_scale,
473 + v_scale,
474 )
475
476 if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/xformers.py
+3
@@ -604,6 +604,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
604 query,
605 key,
606 value,
607 + self.kv_cache_dtype,
608 key_cache,
609 value_cache,
610 prefill_meta.block_tables,
@@ -613,6 +614,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
614 prefill_meta.max_query_len,
615 self.alibi_slopes,
616 self.sliding_window,
617 + k_scale,
618 + v_scale,
619 )
620 assert output[:num_prefill_tokens].shape == out.shape
621 output[:num_prefill_tokens] = out
vllm/attention/ops/ipex_attn.py
+1
@@ -90,6 +90,7 @@ class PagedAttention:
90 query: torch.Tensor,
91 key: torch.Tensor,
92 value: torch.Tensor,
93 + kv_cache_dtype: str,
94 key_cache: torch.Tensor,
95 value_cache: torch.Tensor,
96 block_tables: torch.Tensor,
vllm/attention/ops/paged_attn.py
+6
@@ -194,6 +194,7 @@ class PagedAttention:
194 query: torch.Tensor,
195 key: torch.Tensor,
196 value: torch.Tensor,
197 + kv_cache_dtype: str,
198 key_cache: torch.Tensor,
199 value_cache: torch.Tensor,
200 block_tables: torch.Tensor,
@@ -203,6 +204,8 @@ class PagedAttention:
204 max_query_len: int,
205 alibi_slopes: Optional[torch.Tensor],
206 sliding_window: Optional[int],
207 + k_scale: float,
208 + v_scale: float,
209 ) -> torch.Tensor:
210 output = torch.empty_like(query)
211 context_attention_fwd(
@@ -210,6 +213,7 @@ class PagedAttention:
213 key,
214 value,
215 output,
216 + kv_cache_dtype,
217 key_cache,
218 value_cache,
219 block_tables,
@@ -218,6 +222,8 @@ class PagedAttention:
222 seq_lens_tensor,
223 context_lens,
224 max_query_len,
225 + k_scale,
226 + v_scale,
227 alibi_slopes,
228 sliding_window,
229 )
vllm/attention/ops/prefix_prefill.py
+73 -24
@@ -18,6 +18,8 @@ if triton.__version__ >= "2.1.0":
18 V_cache,
19 B_Loc,
20 sm_scale,
21 + k_scale,
22 + v_scale,
23 B_Start_Loc,
24 B_Seqlen,
25 B_Ctxlen,
@@ -117,10 +119,15 @@ if triton.__version__ >= "2.1.0":
119 cur_kv_head * stride_v_cache_h +
120 offs_d[None, :] * stride_v_cache_d +
121 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
120 - k = tl.load(K_cache + off_k,
121 - mask=dim_mask[:, None] &
122 - ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
123 - other=0.0) # [D,N]
122 + k_load = tl.load(K_cache + off_k,
123 + mask=dim_mask[:, None] &
124 + ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
125 + other=0.0) # [D,N]
126 +
127 + if k_load.dtype.is_fp8():
128 + k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
129 + else:
130 + k = k_load
131
132 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
133 qk += tl.dot(q, k)
@@ -161,12 +168,16 @@ if triton.__version__ >= "2.1.0":
168 acc_scale = l_i / l_i_new * alpha
169 acc = acc * acc_scale[:, None]
170 # update acc
164 - v = tl.load(V_cache + off_v,
165 - mask=dim_mask[None, :] &
166 - ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
167 - other=0.0) # [N,D]
168 -
171 + v_load = tl.load(V_cache + off_v,
172 + mask=dim_mask[None, :] &
173 + ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
174 + other=0.0) # [N,D]
175 + if v_load.dtype.is_fp8():
176 + v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
177 + else:
178 + v = v_load
179 p = p.to(v.dtype)
180 +
181 acc += tl.dot(p, v)
182 # # update m_i and l_i
183 l_i = l_i_new
@@ -225,8 +236,8 @@ if triton.__version__ >= "2.1.0":
236 mask=dim_mask[None, :] &
237 ((start_n + offs_n[:, None]) < cur_batch_query_len),
238 other=0.0)
228 -
239 p = p.to(v.dtype)
240 +
241 acc += tl.dot(p, v)
242 # update m_i and l_i
243 l_i = l_i_new
@@ -336,7 +347,6 @@ if triton.__version__ >= "2.1.0":
347 k = tl.load(K_cache + off_k,
348 mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
349 other=0.0)
339 -
350 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
351 qk += tl.dot(q, k)
352 qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
@@ -442,6 +452,8 @@ if triton.__version__ >= "2.1.0":
452 V_cache,
453 B_Loc,
454 sm_scale,
455 + k_scale,
456 + v_scale,
457 B_Start_Loc,
458 B_Seqlen,
459 B_Ctxlen,
@@ -537,10 +549,15 @@ if triton.__version__ >= "2.1.0":
549 cur_kv_head * stride_v_cache_h +
550 offs_d[None, :] * stride_v_cache_d +
551 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
540 - k = tl.load(K_cache + off_k,
541 - mask=dim_mask[:, None] &
542 - ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
543 - other=0.0) # [D,N]
552 + k_load = tl.load(K_cache + off_k,
553 + mask=dim_mask[:, None] &
554 + ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
555 + other=0.0) # [D,N]
556 +
557 + if k_load.dtype.is_fp8():
558 + k = (k_load.to(tl.float32) * k_scale).to(q.dtype)
559 + else:
560 + k = k_load
561
562 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
563 qk += tl.dot(q, k)
@@ -573,12 +590,16 @@ if triton.__version__ >= "2.1.0":
590 # acc_scale = l_i / l_i_new * alpha
591 acc = acc * acc_scale[:, None]
592 # update acc
576 - v = tl.load(V_cache + off_v,
577 - mask=dim_mask[None, :] &
578 - ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
579 - other=0.0)
580 -
593 + v_load = tl.load(V_cache + off_v,
594 + mask=dim_mask[None, :] &
595 + ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
596 + other=0.0)
597 + if v_load.dtype.is_fp8():
598 + v = (v_load.to(tl.float32) * v_scale).to(q.dtype)
599 + else:
600 + v = v_load
601 p = p.to(v.dtype)
602 +
603 acc += tl.dot(p, v, allow_tf32=False)
604 # update m_i and l_i
605 l_i = l_i_new
@@ -650,8 +671,8 @@ if triton.__version__ >= "2.1.0":
671 ((start_n + offs_n[:, None]) <
672 cur_batch_seq_len - cur_batch_ctx_len),
673 other=0.0)
653 -
674 p = p.to(v.dtype)
675 +
676 acc += tl.dot(p, v, allow_tf32=False)
677 # update m_i and l_i
678 l_i = l_i_new
@@ -675,6 +696,7 @@ if triton.__version__ >= "2.1.0":
696 k,
697 v,
698 o,
699 + kv_cache_dtype: str,
700 k_cache,
701 v_cache,
702 b_loc,
@@ -682,17 +704,41 @@ if triton.__version__ >= "2.1.0":
704 b_seq_len,
705 b_ctx_len,
706 max_input_len,
707 + k_scale: float = 1.0,
708 + v_scale: float = 1.0,
709 alibi_slopes=None,
710 sliding_window=None):
711
712 cap = current_platform.get_device_capability()
713 BLOCK = 128 if cap[0] >= 8 else 64
714 + NUM_WARPS = 8
715
716 # need to reduce num. blocks when using fp32
717 # due to increased use of GPU shared memory
718 if q.dtype is torch.float32:
719 BLOCK = BLOCK // 2
720
721 + # Conversion of FP8 Tensor from uint8 storage to
722 + # appropriate torch.dtype for interpretation by Triton
723 + if "fp8" in kv_cache_dtype:
724 + assert (k_cache.dtype == torch.uint8)
725 + assert (v_cache.dtype == torch.uint8)
726 +
727 + if kv_cache_dtype in ("fp8", "fp8_e4m3"):
728 + target_dtype = torch.float8_e4m3fn
729 + elif kv_cache_dtype == "fp8_e5m2":
730 + target_dtype = torch.float8_e5m2
731 + else:
732 + raise ValueError("Unsupported FP8 dtype:", kv_cache_dtype)
733 +
734 + k_cache = k_cache.view(target_dtype)
735 + v_cache = v_cache.view(target_dtype)
736 +
737 + if (k_cache.dtype == torch.uint8
738 + or v_cache.dtype == torch.uint8 and kv_cache_dtype == "auto"):
739 + raise ValueError("kv_cache_dtype='auto' unsupported for\
740 + FP8 KV Cache prefill kernel")
741 +
742 # shape constraints
743 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
744 assert Lq == Lk and Lk == Lv
@@ -709,7 +755,6 @@ if triton.__version__ >= "2.1.0":
755 if sliding_window is None or sliding_window <= 0:
756 sliding_window = 0
757
712 - num_warps = 8 if Lk <= 64 else 8
758 if alibi_slopes is not None:
759 _fwd_kernel_alibi[grid](
760 q,
@@ -719,6 +764,8 @@ if triton.__version__ >= "2.1.0":
764 v_cache,
765 b_loc,
766 sm_scale,
767 + k_scale,
768 + v_scale,
769 b_start_loc,
770 b_seq_len,
771 b_ctx_len,
@@ -757,7 +804,7 @@ if triton.__version__ >= "2.1.0":
804 BLOCK_DMODEL=Lk,
805 BLOCK_DMODEL_PADDED=Lk_padded,
806 BLOCK_N=BLOCK,
760 - num_warps=num_warps,
807 + num_warps=NUM_WARPS,
808 num_stages=1,
809 )
810 return
@@ -770,6 +817,8 @@ if triton.__version__ >= "2.1.0":
817 v_cache,
818 b_loc,
819 sm_scale,
820 + k_scale,
821 + v_scale,
822 b_start_loc,
823 b_seq_len,
824 b_ctx_len,
@@ -807,7 +856,7 @@ if triton.__version__ >= "2.1.0":
856 BLOCK_DMODEL_PADDED=Lk_padded,
857 BLOCK_N=BLOCK,
858 SLIDING_WINDOW=sliding_window,
810 - num_warps=num_warps,
859 + num_warps=NUM_WARPS,
860 num_stages=1,
861 )
862 return
vllm/config.py
-4
@@ -545,10 +545,6 @@ class CacheConfig:
545 raise NotImplementedError(
546 "Prefix caching is not supported with sliding window. "
547 "Run with --disable-sliding-window to use prefix caching.")
548 - if self.cache_dtype == "fp8":
549 - raise NotImplementedError(
550 - "Prefix caching is not supported for fp8 cache_dtype. "
551 - "Run with --kv-cache-dtype auto to use prefix caching.")
548
549 def verify_with_parallel_config(
550 self,