A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 13:00 UTC
[Bugfix] Respect explicit --kv-cache-dtype over checkpoint kv_cache_scheme (#42782)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 15, 2026 at 20:15 UTC
852f567444cf8c206219edb7b2c42aec55fc41cf
1 file changed
+7
-2
vllm/model_executor/layers/attention/attention.py
+7
-2
@@ -228,9 +228,14 @@ class Attention(nn.Module, AttentionLayerBase):
228
kv_cache_dtype = "auto"
229
calculate_kv_scales = False
230
231
- # llm-compressor mdls need to set cache_dtype to "fp8" manually.
231
+ # llm-compressor models declare an FP8 KV-cache scheme in their
232
+ # checkpoint config. Honor it only when the user did not explicitly
233
+ # pick a kv_cache_dtype; an explicit choice (e.g. bfloat16) must win.
234
+ # The "auto" case is normally resolved upstream in
235
+ # resolve_kv_cache_dtype_string, but we re-apply here defensively in
236
+ # case anything bypassed that path.
237
kv_cache_scheme = getattr(quant_config, "kv_cache_scheme", None)
233
- if kv_cache_scheme is not None:
238
+ if kv_cache_scheme is not None and kv_cache_dtype == "auto":
239
kv_cache_dtype = "fp8"
240
calculate_kv_scales = False
241
if cache_config is not None: