A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
Update modelopt KV cache quantization resolution to new scheme (#31895)
Signed-off-by: Roi Koren <roik@nvidia.com>
roikoren755 committed
Jan 10, 2026 at 06:54 UTC
0c9614876e17680474c240cf9f3138a408a6ec5b
1 file changed
+21
-2
vllm/utils/torch_utils.py
+21
-2
@@ -219,9 +219,28 @@ def get_kv_cache_quant_algo_string(quant_cfg: dict[str, Any]) -> str | None:
219
if quant_method.startswith("modelopt"):
220
quantization_inner = quant_cfg.get("quantization", quant_cfg)
221
# Check if quant config is specified and use kv cache quant algo
222
- kv_algo = quantization_inner.get("kv_cache_quant_algo") or quant_cfg.get(
223
- "kv_cache_quant_algo"
222
+ kv_algo = (
223
+ quantization_inner.get("kv_cache_scheme")
224
+ or quant_cfg.get("kv_cache_scheme")
225
+ or quantization_inner.get("kv_cache_quant_algo")
226
+ or quant_cfg.get("kv_cache_quant_algo")
227
)
228
+ if isinstance(kv_algo, dict):
229
+ if (
230
+ kv_algo.get("dynamic") is False
231
+ and kv_algo.get("num_bits") == 8
232
+ and kv_algo.get("type") == "float"
233
+ ):
234
+ kv_algo = "fp8"
235
+ else:
236
+ # Unknown/unsupported format - return "auto" as safe fallback
237
+ logger.warning(
238
+ "WARNING: Unknown kv_cache_quant_algo '%s' in model "
239
+ "config. Supported values: %s. Falling back to 'auto'.",
240
+ f"{kv_algo}",
241
+ list(MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP.keys()),
242
+ )
243
+ return "auto"
244
if isinstance(kv_algo, str):
245
kv_algo_lower = kv_algo.lower()
246