A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Squelch MLA warning for Compressed-Tensors Models (#12704)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
Feb 3, 2025 at 16:29 UTC
6dd5e52823cc0ca8ddc9c4377d29ead37cc09a95
1 file changed
+4
-2
vllm/config.py
+4
-2
@@ -986,6 +986,9 @@ class ModelConfig:
986
987
@property
988
def use_mla(self) -> bool:
989
+ if not self.is_deepseek_mla or envs.VLLM_MLA_DISABLE:
990
+ return False
991
+
992
if self.quantization is not None and self.quantization not in [\
993
"fp8", "compressed-tensors"]:
994
logger.warning(
@@ -1012,8 +1015,7 @@ class ModelConfig:
1015
quant_config)
1016
return False
1017
1015
- use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
1016
- return use_mla
1018
+ return True
1019
1020
@property
1021
def supported_runner_types(self) -> Set[RunnerType]: