A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 00:30 UTC
[Kimi-Linear] Correct prefixes and add compatibility to AWQ quants (#27834)
Signed-off-by: toncao <cpatonn@gmail.com> Co-authored-by: toncao <cpatonn@gmail.com>
toncao committed
Oct 31, 2025 at 16:36 UTC
e5ef4dfc11abfc44494963b85ced1c79d1d5efea
1 file changed
+2
-1
vllm/model_executor/models/kimi_linear.py
+2
-1
@@ -155,6 +155,7 @@ class KimiMoE(nn.Module):
155
hidden_act=config.hidden_act,
156
quant_config=quant_config,
157
reduce_results=False,
158
+ prefix=f"{prefix}.shared_experts",
159
)
160
161
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -340,7 +341,7 @@ class KimiDecoderLayer(nn.Module):
341
self.block_sparse_moe = KimiMoE(
342
config=config,
343
quant_config=quant_config,
343
- prefix=f"{prefix}.mlp",
344
+ prefix=f"{prefix}.block_sparse_moe",
345
)
346
self.mlp = self.block_sparse_moe
347
else: