A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Switch fp8 layers to use the CUTLASS kernels (#5183)
Switching from torch._scaled_mm to vLLM's cutlass fp8 kernels when supported as we are seeing 5-15% improvement in e2e performance on neuralmagic/Meta-Llama-3-8B-Instruct-FP8 see https://docs.google.com/spreadsheets/d/1GiAnmzyGHgZ6zL_LDSTm35Bdrt4A8AaFEurDlISYYA4/ for some quick e2e benchmarks and #5144 for comparisons across different GEMM sizes.
Tyler Michael Smith committed
Jun 7, 2024 at 04:42 UTC
8d75fe48ca5f46b7af0f5201d8500b9604eed769
2 files changed
+52
-18
vllm/_custom_ops.py
+2
-2
@@ -179,7 +179,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
179
180
# cutlass
181
def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
182
- a_scales: torch.Tensor, b_scales: torch.Tensor,
182
+ scale_a: torch.Tensor, scale_b: torch.Tensor,
183
out_dtype: Type[torch.dtype]) -> torch.Tensor:
184
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
185
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
@@ -188,7 +188,7 @@ def cutlass_scaled_mm_dq(a: torch.Tensor, b: torch.Tensor,
188
n = b.shape[1]
189
out = torch.empty((m, n), dtype=out_dtype, device=a.device)
190
191
- vllm_ops.cutlass_scaled_mm_dq(out, a, b, a_scales, b_scales)
191
+ vllm_ops.cutlass_scaled_mm_dq(out, a, b, scale_a, scale_b)
192
193
return out
194
vllm/model_executor/layers/quantization/fp8.py
+50
-16
@@ -17,6 +17,24 @@ ACTIVATION_SCHEMES = ["static", "dynamic"]
17
logger = init_logger(__name__)
18
19
20
+def cutlass_fp8_supported() -> bool:
21
+ capability = torch.cuda.get_device_capability()
22
+ capability = capability[0] * 10 + capability[1]
23
+ version = torch.version.cuda
24
+ version = version[0] * 10 + version[1]
25
+
26
+ # CUTLASS FP8 kernels need at least
27
+ # CUDA 12.0 on SM90 systems (Hopper)
28
+ # CUDA 12.4 on SM89 systems (Lovelace)
29
+ gpu_is_supported = False
30
+ if capability >= 900:
31
+ gpu_is_supported = version > 120
32
+ elif capability >= 890:
33
+ gpu_is_supported = version > 124
34
+
35
+ return gpu_is_supported
36
+
37
+
38
class Fp8Config(QuantizationConfig):
39
"""Config class for FP8."""
40
@@ -92,6 +110,7 @@ class Fp8LinearMethod(LinearMethodBase):
110
111
def __init__(self, quant_config: Fp8Config):
112
self.quant_config = quant_config
113
+ self.cutlass_fp8_supported = cutlass_fp8_supported()
114
115
def _create_scale_param(
116
self,
@@ -233,25 +252,40 @@ class Fp8LinearMethod(LinearMethodBase):
252
layer: torch.nn.Module,
253
x: torch.Tensor,
254
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
255
+
256
# ops.scaled_fp8_quant supports both dynamic and static quant.
257
# If dynamic, layer.act_scale is None and x_scale computed from x.
258
# If static, layer.act_scale is scalar and x_scale set to act_scale.
239
- qinput, x_scale = ops.scaled_fp8_quant(x,
240
- layer.act_scale,
241
- batch_dim_padding=17)
242
-
243
- # Fused GEMM_DQ -- note we padded the input above because
244
- # torch._scaled_mm is more performant for matrices with
245
- # batch dimension > 16. Note that this could change
246
- # in the future.
247
- output, _ = torch._scaled_mm(
248
- qinput,
249
- layer.weight,
250
- out_dtype=x.dtype,
251
- scale_a=x_scale,
252
- scale_b=layer.weight_scale,
253
- bias=bias,
254
- )
259
+
260
+ if bias is None and self.cutlass_fp8_supported:
261
+ qinput, x_scale = ops.scaled_fp8_quant(x, layer.act_scale)
262
+
263
+ # Fused GEMM_DQ
264
+ output = ops.cutlass_scaled_mm_dq(
265
+ qinput,
266
+ layer.weight,
267
+ out_dtype=x.dtype,
268
+ scale_a=x_scale,
269
+ scale_b=layer.weight_scale,
270
+ )
271
+
272
+ else:
273
+ qinput, x_scale = ops.scaled_fp8_quant(x,
274
+ layer.act_scale,
275
+ batch_dim_padding=17)
276
+
277
+ # Fused GEMM_DQ -- note we padded the input above because
278
+ # torch._scaled_mm is more performant for matrices with
279
+ # batch dimension > 16. Note that this could change
280
+ # in the future.
281
+ output, _ = torch._scaled_mm(
282
+ qinput,
283
+ layer.weight,
284
+ out_dtype=x.dtype,
285
+ scale_a=x_scale,
286
+ scale_b=layer.weight_scale,
287
+ bias=bias,
288
+ )
289
290
return torch.narrow(output, 0, 0, x.shape[0])
291