A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 02:45 UTC
[ROCm] Improve error handling while loading quantized model on gfx120… (#31715)
Signed-off-by: brian033 <85883730+brian033@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
brian033 committed
Jan 15, 2026 at 20:16 UTC
b89275d018796170acbb91b8bcddbc8b8213025e
1 file changed
+5
-1
vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py
+5
-1
@@ -153,7 +153,11 @@ try:
153
fake_impl=gemm_with_dynamic_quant_fake,
154
dispatch_key=current_platform.dispatch_key,
155
)
156
-except (ImportError, AttributeError):
156
+except (ImportError, AttributeError, RuntimeError):
157
+ logger.warning(
158
+ "AITER is not found or QuarkOCP_MX is not supported on the current "
159
+ "platform. QuarkOCP_MX quantization will not be available."
160
+ )
161
dynamic_mxfp4_quant = gemm_afp4wfp4 = None
162
163