A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Disable fp8 kv-cache by default for DeepSeek V3.2 (#27121)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Lucas Wilkinson committed
Oct 18, 2025 at 18:05 UTC
c2bba690658823be5ec6f1742eb10294d4fa2479
1 file changed
+2
-5
vllm/model_executor/models/config.py
+2
-5
@@ -481,12 +481,9 @@ class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
481
is_v32 = hasattr(hf_config, "index_topk")
482
assert is_v32
483
484
- # For DeepSeekV3.2, we use a custom fp8 format as default (i.e.
485
- # "auto")
484
+ # For DeepSeekV3.2, a custom fp8 format is used when fp8 kv-cache is enabled.
485
cache_config = vllm_config.cache_config
487
- if cache_config.cache_dtype == "auto" or cache_config.cache_dtype.startswith(
488
- "fp8"
489
- ):
486
+ if cache_config.cache_dtype.startswith("fp8"):
487
cache_config.cache_dtype = "fp8_ds_mla"
488
logger.info("Using custom fp8 kv-cache format for DeepSeekV3.2")
489
if cache_config.cache_dtype == "bfloat16":