@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d75fe48ca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Switch fp8 layers to use the CUTLASS kernels (#5183)

Switching from torch._scaled_mm to vLLM's cutlass fp8 kernels when supported as we are seeing 5-15% improvement in e2e performance on neuralmagic/Meta-Llama-3-8B-Instruct-FP8 see https://docs.google.com/spreadsheets/d/1GiAnmzyGHgZ6zL_LDSTm35Bdrt4A8AaFEurDlISYYA4/ for some quick e2e benchmarks and #5144 for comparisons across different GEMM sizes.

Tyler Michael Smith committed Jun 7, 2024 at 04:42 UTC 8d75fe48ca5f46b7af0f5201d8500b9604eed769
2 files changed +52 -18
vllm/_custom_ops.py
+2 -2
@@ -179,7 +179,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
179
180 # cutlass
181 def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
182 - a_scales: torch.Tensor, b_scales: torch.Tensor,
182 + scale_a: torch.Tensor, scale_b: torch.Tensor,
183 out_dtype: Type[torch.dtype]) -> torch.Tensor:
184 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
185 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
@@ -188,7 +188,7 @@ def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
188 n = b.shape[1]
189 out = torch.empty((m, n), dtype=out_dtype, device=a.device)
190
191 - vllm_ops.cutlass_scaled_mm_dq(out, a, b, a_scales, b_scales)
191 + vllm_ops.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
192
193 return out
194
vllm/model_executor/layers/quantization/fp8.py
+50 -16
@@ -17,6 +17,24 @@ ACTIVATION_SCHEMES = ["static", "dynamic"]
17 logger = init_logger(__name__)
18
19
20 +def cutlass_fp8_supported() -> bool:
21 + capability = torch.cuda.get_device_capability()
22 + capability = capability[0] * 10 + capability[1]
23 + version = torch.version.cuda
24 + version = version[0] * 10 + version[1]
25 +
26 + # CUTLASS FP8 kernels need at least
27 + # CUDA 12.0 on SM90 systems (Hopper)
28 + # CUDA 12.4 on SM89 systems (Lovelace)
29 + gpu_is_supported = False
30 + if capability >= 900:
31 + gpu_is_supported = version > 120
32 + elif capability >= 890:
33 + gpu_is_supported = version > 124
34 +
35 + return gpu_is_supported
36 +
37 +
38 class Fp8Config(QuantizationConfig):
39 """Config class for FP8."""
40
@@ -92,6 +110,7 @@ class Fp8LinearMethod(LinearMethodBase):
110
111 def __init__(self, quant_config: Fp8Config):
112 self.quant_config = quant_config
113 + self.cutlass_fp8_supported = cutlass_fp8_supported()
114
115 def _create_scale_param(
116 self,
@@ -233,25 +252,40 @@ class Fp8LinearMethod(LinearMethodBase):
252 layer: torch.nn.Module,
253 x: torch.Tensor,
254 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
255 +
256 # ops.scaled_fp8_quant supports both dynamic and static quant.
257 # If dynamic, layer.act_scale is None and x_scale computed from x.
258 # If static, layer.act_scale is scalar and x_scale set to act_scale.
239 - qinput, x_scale = ops.scaled_fp8_quant(x,
240 - layer.act_scale,
241 - batch_dim_padding=17)
242 -
243 - # Fused GEMM_DQ -- note we padded the input above because
244 - # torch._scaled_mm is more performant for matrices with
245 - # batch dimension > 16. Note that this could change
246 - # in the future.
247 - output, _ = torch._scaled_mm(
248 - qinput,
249 - layer.weight,
250 - out_dtype=x.dtype,
251 - scale_a=x_scale,
252 - scale_b=layer.weight_scale,
253 - bias=bias,
254 - )
259 +
260 + if bias is None and self.cutlass_fp8_supported:
261 + qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
262 +
263 + # Fused GEMM_DQ
264 + output = ops.cutlass_scaled_mm_dq(
265 + qinput,
266 + layer.weight,
267 + out_dtype=x.dtype,
268 + scale_a=x_scale,
269 + scale_b=layer.weight_scale,
270 + )
271 +
272 + else:
273 + qinput, x_scale = ops.scaled_fp8_quant(x,
274 + layer.act_scale,
275 + batch_dim_padding=17)
276 +
277 + # Fused GEMM_DQ -- note we padded the input above because
278 + # torch._scaled_mm is more performant for matrices with
279 + # batch dimension > 16. Note that this could change
280 + # in the future.
281 + output, _ = torch._scaled_mm(
282 + qinput,
283 + layer.weight,
284 + out_dtype=x.dtype,
285 + scale_a=x_scale,
286 + scale_b=layer.weight_scale,
287 + bias=bias,
288 + )
289
290 return torch.narrow(output, 0, 0, x.shape[0])
291