A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix: AWQ Marlin get_quant_method does not recognize "modules_to_not_convert" (#21888)
Signed-off-by: JunHowie <JunHowie@aliyun.com> Co-authored-by: JunHowie <JunHowie@aliyun.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Jun-Howie committed
Aug 12, 2025 at 17:03 UTC
1ece7f30baa9d94ff57e13d851725acf657a9690
1 file changed
+6
-2
vllm/model_executor/layers/quantization/awq_marlin.py
+6
-2
@@ -10,7 +10,8 @@ import vllm.model_executor.layers.fused_moe # noqa
10
from vllm import _custom_ops as ops
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.fused_moe.layer import (
13
- FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
13
+ FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported,
14
+ UnquantizedFusedMoEMethod)
15
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
16
UnquantizedLinearMethod,
17
set_weight_attrs)
@@ -141,6 +142,9 @@ class AWQMarlinConfig(QuantizationConfig):
142
elif isinstance(layer, FusedMoE):
143
from vllm.model_executor.layers.quantization.moe_wna16 import (
144
MoeWNA16Config)
145
+ if is_layer_skipped_awq(
146
+ prefix, getattr(self, "modules_to_not_convert", [])):
147
+ return UnquantizedFusedMoEMethod(layer.moe_config)
148
if not check_moe_marlin_supports_layer(layer, self.group_size):
149
logger.warning_once(
150
f"Layer '{prefix}' is not supported by AWQMoeMarlin. "
@@ -520,4 +524,4 @@ class AWQMoEMethod(FusedMoEMethodBase):
524
expert_map=expert_map,
525
w1_zeros=layer.w13_qzeros,
526
w2_zeros=layer.w2_qzeros,
523
- workspace=layer.workspace)
527
+ workspace=layer.workspace)
\ No newline at end of file