A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 03:45 UTC
[Quantization] fix marlin w8a8 check (#30961)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Jinzhen Lin committed
Dec 19, 2025 at 23:33 UTC
5fbfa8d9ef15948599631baeb91e8220b2ee9bcc
1 file changed
+3
-6
vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py
+3
-6
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
11
marlin_make_workspace_new,
12
marlin_permute_bias,
13
marlin_permute_scales,
14
- marlin_quant_input,
14
should_use_atomic_add_reduce,
15
)
16
from vllm.model_executor.utils import replace_parameter
@@ -63,13 +62,11 @@ def apply_fp8_marlin_linear(
62
inputs = reshaped_x
63
a_scales = None
64
if input_dtype is not None and input_dtype.itemsize == 1:
66
- if input_dtype != torch.float8_e4m3fn:
67
- raise RuntimeError("FP8 weight + INT8 activation is not supported.")
68
-
69
- inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
65
+ # inputs, a_scales = marlin_quant_input(inputs, torch.float8_e4m3fn)
66
+ raise RuntimeError("Marlin W8A8 is not supported.")
67
68
output = ops.gptq_marlin_gemm(
72
- a=reshaped_x,
69
+ a=inputs,
70
c=None,
71
b_q_weight=weight,
72
b_bias=bias,