A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[BugFix] Fix DeepSeek-V3.1 + DeepGEMM incompatible scale shapes (#32361)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Eldar Kurtić <8884008+eldarkurtic@users.noreply.github.com>
Lucas Wilkinson committed
Jan 14, 2026 at 23:32 UTC
2c9b4cf5bf844de0471f77e6579e16c7bc3ee0d0
1 file changed
+3
vllm/model_executor/layers/quantization/utils/quant_utils.py
+3
@@ -300,6 +300,9 @@ def get_and_maybe_dequant_weights(
300
if (
301
isinstance(layer.quant_method, Fp8LinearMethod)
302
and not layer.quant_method.use_marlin
303
+ # DeepGEMM transforms the scales using `transform_sf_into_required_layout` into
304
+ # a layout that is not compatible with `scaled_dequantize`.
305
+ and not layer.quant_method.use_deep_gemm
306
):
307
weight_scales = get_attribute_fallback(
308
layer, ["weight_scale", "weight_scale_inv"]