Fix DeepSeek V4 FlashMLA auto KV cache dtype (#45091)
Signed-off-by: elehayym <huangxh1618@gmail.com> Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: elehayym <huangxh1618@gmail.com> Co-authored-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
elehayym committed
Sep 4, 2026 at 03:29 UTC
fc8f10792c59011447b56b41d4c8b1048673e939
1 file changed
+9
-4
vllm/models/deepseek_v4/attention.py
+9
-4
@@ -102,10 +102,15 @@ def _resolve_dsv4_kv_cache_dtype(
102
"""
103
if use_fp8_ds_mla_layout:
104
# fp8_ds_mla block format: UE8M0 block-scaled fp8 packed as uint8.
105
- assert kv_cache_dtype.startswith("fp8"), (
106
- f"DeepseekV4 fp8_ds_mla layout only supports fp8 kv-cache, "
107
- f"got {kv_cache_dtype}"
108
- )
105
+ if kv_cache_dtype == "auto":
106
+ kv_cache_dtype = "fp8"
107
+ if not kv_cache_dtype.startswith("fp8"):
108
+ raise ValueError(
109
+ "DeepseekV4 fp8_ds_mla layout only supports fp8 "
110
+ f"kv-cache, got {kv_cache_dtype}. Please set "
111
+ "`--kv-cache-dtype fp8` or select a backend that supports "
112
+ "bfloat16 KV cache."
113
+ )
114
if kv_cache_dtype != "fp8_ds_mla":
115
if cache_config is not None:
116
cache_config.cache_dtype = "fp8_ds_mla"