A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix: Correct FusedMoE layer reference in auto_round quantization (#24818)
Signed-off-by: David-Wen <18927700430@163.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
David-Wen committed
Sep 20, 2025 at 04:44 UTC
ddc9048394ae6294d0db7fd67270efea59c3a065
1 file changed
+2
-2
vllm/model_executor/layers/quantization/auto_round.py
+2
-2
@@ -241,7 +241,7 @@ class AutoRoundConfig(QuantizationConfig):
241
242
if isinstance(layer, FusedMoE):
243
if use_marlin:
244
- return AWQMoEMethod(quant_args_marlin, layer.moe)
244
+ return AWQMoEMethod(quant_args_marlin, layer.moe_config)
245
from vllm.model_executor.layers.quantization.moe_wna16 import (
246
MoeWNA16Config)
247
@@ -327,7 +327,7 @@ class AutoRoundConfig(QuantizationConfig):
327
328
if isinstance(layer, FusedMoE):
329
if use_marlin:
330
- return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe)
330
+ return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
331
else:
332
from vllm.model_executor.layers.quantization.moe_wna16 import (
333
MoeWNA16Config)