A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 16:30 UTC
[Bugfix] Fix KV Scale loading for MLA Models (#35430)
Signed-off-by: Pavani Majety <pmajety@nvidia.com>
Pavani Majety committed
Feb 26, 2026 at 15:38 UTC
6283021142bbf5ee324395dad5e80b8661400329
1 file changed
+2
-2
vllm/model_executor/layers/quantization/modelopt.py
+2
-2
@@ -12,7 +12,7 @@ from vllm.logger import init_logger
12
from vllm.model_executor.kernels.linear import (
13
init_fp8_linear_kernel,
14
)
15
-from vllm.model_executor.layers.attention import Attention
15
+from vllm.model_executor.layers.attention import Attention, MLAAttention
16
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
17
from vllm.model_executor.layers.fused_moe.config import (
18
FusedMoEConfig,
@@ -183,7 +183,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
183
self, layer: torch.nn.Module, prefix: str
184
) -> "QuantizeMethodBase | None":
185
# handle kv-cache first so we can focus only on weight quantization thereafter
186
- if isinstance(layer, Attention):
186
+ if isinstance(layer, (Attention, MLAAttention)):
187
return self.KVCacheMethodCls(self)
188
189
# handle exclusion