A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Fix noisy warning for uncalibrated q_scale/p_scale (#17414)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 8, 2025 at 15:56 UTC
4f605a6de5940b52e9f83cd79f542b6732535252
1 file changed
+5
-4
vllm/model_executor/layers/quantization/kv_cache.py
+5
-4
@@ -124,11 +124,12 @@ class BaseKVCacheMethod(QuantizeMethodBase):
124
# These are used in the final Attention.forward()
125
layer._q_scale.copy_(q_scale)
126
layer._prob_scale.copy_(prob_scale)
127
- if q_scale == 1.0 or prob_scale == 1.0:
127
+ if layer.kv_cache_dtype == "fp8" and (q_scale == 1.0
128
+ or prob_scale == 1.0):
129
logger.warning_once(
129
- f"Using Q scale {q_scale} and prob scale {prob_scale} "
130
- "with fp8 attention. This may cause accuracy issues. "
131
- "Please make sure Q/prob scaling factors are "
130
+ f"Using uncalibrated q_scale {q_scale} and/or prob_scale "
131
+ f"{prob_scale} with fp8 attention. This may cause accuracy "
132
+ "issues. Please make sure q/prob scaling factors are "
133
"available in the fp8 checkpoint.")
134
135
del layer.k_scale