A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix Auto_Round Quatization Loading on SM75 and Lower GPUs (#24217)
Signed-off-by: RoadToNowhereX <37441177+RoadToNowhereX@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
RoadToNowhereX committed
Sep 10, 2025 at 23:22 UTC
c0bd6a684aee2b47fe52b75ec97379e40ca5d36c
1 file changed
+2
-1
vllm/model_executor/layers/quantization/auto_round.py
+2
-1
@@ -327,6 +327,8 @@ class AutoRoundConfig(QuantizationConfig):
327
328
if isinstance(layer, FusedMoE):
329
if use_marlin:
330
+ return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe)
331
+ else:
332
from vllm.model_executor.layers.quantization.moe_wna16 import (
333
MoeWNA16Config)
334
@@ -339,7 +341,6 @@ class AutoRoundConfig(QuantizationConfig):
341
}
342
return MoeWNA16Config.from_config(config).get_quant_method(
343
layer, prefix)
342
- return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe)
344
345
if isinstance(layer, (LinearBase, ParallelLMHead)):
346
if use_marlin: