A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[ROCM] Enable CompressedTensorsWNA16 (#27187)
Signed-off-by: JartX <sagformas@epdcenter.es>
JartX committed
Oct 21, 2025 at 16:43 UTC
ba09652de28dcf43fc0b55b50f400eac768e800b
1 file changed
+4
-1
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+4
-1
@@ -142,7 +142,10 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
142
# group_size=None means channelwise
143
group_size = weight_quant.group_size or -1
144
# Prefer to use the MarlinMoE kernel when it is supported.
145
- if not check_moe_marlin_supports_layer(layer, group_size):
145
+ if (
146
+ not check_moe_marlin_supports_layer(layer, group_size)
147
+ or current_platform.is_rocm()
148
+ ):
149
if (
150
weight_quant.strategy == QuantizationStrategy.GROUP
151
and weight_quant.actorder