A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Kernel] Use CUTLASS kernels for the FP8 layers with Bias (#6270)
Tyler Michael Smith committed
Jul 15, 2024 at 13:05 UTC
c8fd97f26d05aff5a4603177c75aaccf4e6de11b
1 file changed
+3
-2
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3
-2
@@ -112,7 +112,7 @@ def apply_fp8_linear(
112
# If dynamic, layer.input_scale is None and x_scale computed from x.
113
# If static, layer.input_scale is scalar and x_scale is input_scale.
114
115
- if bias is None and cutlass_fp8_supported:
115
+ if cutlass_fp8_supported:
116
qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
117
118
# Fused GEMM_DQ
@@ -120,7 +120,8 @@ def apply_fp8_linear(
120
weight,
121
out_dtype=input.dtype,
122
scale_a=x_scale,
123
- scale_b=weight_scale)
123
+ scale_b=weight_scale,
124
+ bias=bias)
125
126
else:
127
qinput, x_scale = ops.scaled_fp8_quant(input,