A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[Bugfix] Fixes for new marlin moe usage (#18017)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 12, 2025 at 23:50 UTC
1df491c522c92c3b15dea4a4cd92c437bbda9f3f
2 files changed
+5
-4
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+3
-2
@@ -57,9 +57,10 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
57
"input_activations")
58
59
if quant_config._is_wNa16_group_channel(weight_quant, input_quant):
60
+ # group_size=None means channelwise
61
+ group_size = weight_quant.group_size or -1
62
# Prefer to use the MarlinMoE kernel when it is supported.
61
- if not check_moe_marlin_supports_layer(layer,
62
- weight_quant.group_size):
63
+ if not check_moe_marlin_supports_layer(layer, group_size):
64
if (weight_quant.strategy in QuantizationStrategy.GROUP and
65
weight_quant.actorder in (ActivationOrdering.GROUP,
66
ActivationOrdering.DYNAMIC)):
vllm/model_executor/layers/quantization/gptq_marlin.py
+2
-2
@@ -610,9 +610,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
610
activation: str = "silu",
611
) -> torch.Tensor:
612
assert activation == "silu", "Only SiLU activation is supported."
613
- if apply_router_weight_on_input is not None:
613
+ if apply_router_weight_on_input:
614
raise NotImplementedError(
615
- "Apply router weight on input is not supported for"
615
+ "Apply router weight on input is not supported for "
616
"fused Marlin MoE method.")
617
618
topk_weights, topk_ids = FusedMoE.select_experts(