@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8fd97f26d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Kernel] Use CUTLASS kernels for the FP8 layers with Bias (#6270)

Tyler Michael Smith committed Jul 15, 2024 at 13:05 UTC c8fd97f26d05aff5a4603177c75aaccf4e6de11b
1 file changed +3 -2
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+3 -2
@@ -112,7 +112,7 @@ def apply_fp8_linear(
112 # If dynamic, layer.input_scale is None and x_scale computed from x.
113 # If static, layer.input_scale is scalar and x_scale is input_scale.
114
115 - if bias is None and cutlass_fp8_supported:
115 + if cutlass_fp8_supported:
116 qinput, x_scale = ops.scaled_fp8_quant(input, input_scale)
117
118 # Fused GEMM_DQ
@@ -120,7 +120,8 @@ def apply_fp8_linear(
120 weight,
121 out_dtype=input.dtype,
122 scale_a=x_scale,
123 - scale_b=weight_scale)
123 + scale_b=weight_scale,
124 + bias=bias)
125
126 else:
127 qinput, x_scale = ops.scaled_fp8_quant(input,