A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[UX] Only show FP4 Marlin fallback warning for w4a4 models (#36806)
Co-authored-by: Claude <noreply@anthropic.com>
Michael Goin committed
Mar 12, 2026 at 10:19 UTC
57431d8231235cdae89e71b4024f611858c47372
3 files changed
+12
-14
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+6
@@ -324,6 +324,12 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
324
)
325
delattr(layer, "w2_weight_packed")
326
327
+ logger.warning_once(
328
+ "Your GPU does not have native support for FP4 computation but "
329
+ "FP4 quantization is being used. Weight-only FP4 compression "
330
+ "will be used leveraging the Marlin kernel. This may degrade "
331
+ "performance for compute-heavy workloads."
332
+ )
333
prepare_moe_fp4_layer_for_marlin(layer)
334
335
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py
-14
@@ -147,13 +147,6 @@ def apply_fp4_marlin_linear(
147
def prepare_fp4_layer_for_marlin(
148
layer: torch.nn.Module, input_dtype: torch.dtype | None = None
149
) -> None:
150
- logger.warning_once(
151
- "Your GPU does not have native support for FP4 computation but "
152
- "FP4 quantization is being used. Weight-only FP4 compression will "
153
- "be used leveraging the Marlin kernel. This may degrade "
154
- "performance for compute-heavy workloads."
155
- )
156
-
150
is_nvfp4 = hasattr(layer, "weight_global_scale")
151
if input_dtype is not None and input_dtype.itemsize == 1:
152
if is_nvfp4:
@@ -335,13 +328,6 @@ def prepare_nvfp4_moe_layer_for_marlin(
328
def prepare_moe_fp4_layer_for_marlin(
329
layer: torch.nn.Module, input_dtype: torch.dtype | None = None
330
) -> None:
338
- logger.warning_once(
339
- "Your GPU does not have native support for FP4 computation but "
340
- "FP4 quantization is being used. Weight-only FP4 compression will "
341
- "be used leveraging the Marlin kernel. This may degrade "
342
- "performance for compute-heavy workloads."
343
- )
344
-
331
is_nvfp4 = hasattr(layer, "w13_weight_scale_2")
332
if input_dtype is not None and input_dtype.itemsize == 1:
333
if is_nvfp4:
vllm/model_executor/layers/quantization/utils/nvfp4_utils.py
+6
@@ -141,6 +141,12 @@ def convert_to_nvfp4_linear_kernel_format(
141
layer.weights_padding_cols = 0
142
143
if backend == NvFp4LinearBackend.MARLIN:
144
+ logger.warning_once(
145
+ "Your GPU does not have native support for FP4 computation but "
146
+ "FP4 quantization is being used. Weight-only FP4 compression "
147
+ "will be used leveraging the Marlin kernel. This may degrade "
148
+ "performance for compute-heavy workloads."
149
+ )
150
prepare_fp4_layer_for_marlin(layer)
151
elif backend == NvFp4LinearBackend.FLASHINFER_TRTLLM:
152
weight, weight_scale = prepare_weights_for_nvfp4_flashinfer_trtllm(