A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[Quantization] add marlin w4a8/w8a8 check (#31061)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Jinzhen Lin committed
Dec 21, 2025 at 05:58 UTC
7c73ceb5812ace65e0d1b6ada3622b8b9f0400c0
3 files changed
+28
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+12
@@ -594,9 +594,15 @@ def apply_awq_marlin_linear(
594
595
a_scales = None
596
if input_dtype == torch.int8:
597
+ assert quant_type == scalar_types.uint4, (
598
+ "W8A8-INT8 is not supported by marlin kernel."
599
+ )
600
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
601
a_scales = a_scales * input_global_scale
602
elif input_dtype == torch.float8_e4m3fn:
603
+ assert quant_type == scalar_types.uint4, (
604
+ "INT8 weight + FP8 activation is not supported."
605
+ )
606
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
607
608
output = ops.gptq_marlin_gemm(
@@ -649,9 +655,15 @@ def apply_rtn_marlin_linear(
655
656
a_scales = None
657
if input_dtype == torch.int8:
658
+ assert quant_type == scalar_types.uint4b8, (
659
+ "W8A8-INT8 is not supported by marlin kernel."
660
+ )
661
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
662
a_scales = a_scales * input_global_scale
663
elif input_dtype == torch.float8_e4m3fn:
664
+ assert quant_type == scalar_types.uint4b8, (
665
+ "INT8 weight + FP8 activation is not supported."
666
+ )
667
reshaped_x, a_scales = marlin_quant_input(reshaped_x, input_dtype)
668
669
output = ops.gptq_marlin_gemm(
vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py
+12
@@ -154,6 +154,12 @@ def prepare_fp4_layer_for_marlin(
154
)
155
156
is_nvfp4 = hasattr(layer, "weight_scale_2")
157
+ if input_dtype is not None and input_dtype.itemsize == 1:
158
+ if is_nvfp4:
159
+ raise RuntimeError("NVFP4 weight + INT8/FP8 activation is not supported.")
160
+ elif input_dtype != torch.float8_e4m3fn:
161
+ raise RuntimeError("MXFP4 weight + INT8 activation is not supported.")
162
+
163
group_size = 16 if is_nvfp4 else 32
164
165
part_size_n = layer.output_size_per_partition
@@ -231,6 +237,12 @@ def prepare_moe_fp4_layer_for_marlin(
237
)
238
239
is_nvfp4 = hasattr(layer, "w13_weight_scale_2")
240
+ if input_dtype is not None and input_dtype.itemsize == 1:
241
+ if is_nvfp4:
242
+ raise RuntimeError("NVFP4 weight + INT8/FP8 activation is not supported.")
243
+ elif input_dtype != torch.float8_e4m3fn:
244
+ raise RuntimeError("MXFP4 weight + INT8 activation is not supported.")
245
+
246
group_size = 16 if is_nvfp4 else 32
247
248
e = layer.num_experts
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+4
@@ -99,6 +99,8 @@ def prepare_fp8_layer_for_marlin(
99
"be used leveraging the Marlin kernel. This may degrade "
100
"performance for compute-heavy workloads."
101
)
102
+ if input_dtype is not None and input_dtype.itemsize == 1:
103
+ raise RuntimeError("Marlin W8A8 is not supported.")
104
105
part_size_n = layer.output_size_per_partition
106
part_size_k = layer.input_size_per_partition
@@ -206,6 +208,8 @@ def prepare_moe_fp8_layer_for_marlin(
208
"be used leveraging the Marlin kernel. This may degrade "
209
"performance for compute-heavy workloads."
210
)
211
+ if input_dtype is not None and input_dtype.itemsize == 1:
212
+ raise RuntimeError("Marlin W8A8 is not supported.")
213
214
e = layer.num_experts
215
k = layer.hidden_size