A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 04:45 UTC
add warning when FP8 KV cache misses prefill query quantization (#39752)
Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Albert Cheng (Engrg-Hardware 1) <albecheng@login-lyris02.lyris.clusters.nvidia.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Albert Cheng committed
Apr 14, 2026 at 11:43 UTC
dc8df110bc8a5c80dbb8f1fef02e98deb8aa2b09
1 file changed
+13
vllm/model_executor/layers/attention/mla_attention.py
+13
@@ -1443,6 +1443,19 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
1443
scope="local",
1444
)
1445
return model_dtype
1446
+ elif (
1447
+ is_quantized_kv_cache(vllm_config.cache_config.cache_dtype)
1448
+ and backend_supports_prefill_query_quantization()
1449
+ ):
1450
+ logger.warning_once(
1451
+ "FP8 KV cache is enabled but prefill queries are not "
1452
+ "quantized to FP8. For long-context workloads (ISL >= 4K), "
1453
+ "enabling FP8 prefill attention can significantly optimize "
1454
+ "prefill latency. To enable, add: "
1455
+ '--attention-config \'{"use_prefill_query_quantization"'
1456
+ ": true}'",
1457
+ scope="local",
1458
+ )
1459
1460
return model_dtype
1461