A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 09:30 UTC
[Bugfix] Fix AttributeError when serving MXFP8 models with DeepGEMM installed (#37358)
Signed-off-by: EdalatiAli <aliedalati@cohere.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
EdalatiAli committed
Mar 19, 2026 at 13:58 UTC
daa05bf340cb74b062db727395dce89a7387a832
1 file changed
+4
-2
vllm/model_executor/warmup/deep_gemm_warmup.py
+4
-2
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import (
19
)
20
from vllm.model_executor.layers.linear import LinearBase
21
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
22
+from vllm.model_executor.layers.quantization.mxfp8 import Mxfp8OnlineLinearMethod
23
from vllm.tracing import instrument
24
from vllm.utils.deep_gemm import (
25
fp8_gemm_nt,
@@ -136,8 +137,9 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool:
137
if not (
138
isinstance(module, LinearBase)
139
and isinstance(module.quant_method, Fp8LinearMethod)
139
- and module.quant_method.block_quant
140
- and not module.quant_method.use_marlin
140
+ and not isinstance(module.quant_method, Mxfp8OnlineLinearMethod)
141
+ and getattr(module.quant_method, "block_quant", False)
142
+ and not getattr(module.quant_method, "use_marlin", True)
143
):
144
return False
145