@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6dd5e52823
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Squelch MLA warning for Compressed-Tensors Models (#12704)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Kyle Sayers committed Feb 3, 2025 at 16:29 UTC 6dd5e52823cc0ca8ddc9c4377d29ead37cc09a95
1 file changed +4 -2
vllm/config.py
+4 -2
@@ -986,6 +986,9 @@ class ModelConfig:
986
987 @property
988 def use_mla(self) -> bool:
989 + if not self.is_deepseek_mla or envs.VLLM_MLA_DISABLE:
990 + return False
991 +
992 if self.quantization is not None and self.quantization not in [\
993 "fp8", "compressed-tensors"]:
994 logger.warning(
@@ -1012,8 +1015,7 @@ class ModelConfig:
1015 quant_config)
1016 return False
1017
1015 - use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
1016 - return use_mla
1018 + return True
1019
1020 @property
1021 def supported_runner_types(self) -> Set[RunnerType]: