@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6a2d659d28
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix compute datatype for cutlass 3.x epilogues (#5931)

Tyler Michael Smith committed Jun 28, 2024 at 13:10 UTC 6a2d659d28a9f9f3edbfbae138915d147a9fe79c
2 files changed +70 -59
csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu
+2 -2
@@ -144,14 +144,14 @@ struct ScaledEpilogueBias
144 using ScaleB = typename SUPER::ScaleB;
145
146 using Compute0 = cutlass::epilogue::fusion::Sm90Compute<
147 - cutlass::multiplies, ElementD, ElementD,
147 + cutlass::multiplies, float, float,
148 cutlass::FloatRoundStyle::round_to_nearest>;
149
150 using EVTCompute0 =
151 cutlass::epilogue::fusion::Sm90EVT<Compute0, ScaleB, Accum>;
152
153 using Compute1 = cutlass::epilogue::fusion::Sm90Compute<
154 - cutlass::multiply_add, ElementD, ElementD,
154 + cutlass::multiply_add, ElementD, float,
155 cutlass::FloatRoundStyle::round_to_nearest>;
156
157 using BiasDescriptor =
tests/kernels/test_cutlass.py
+68 -57
@@ -2,7 +2,7 @@
2
3 Run `pytest tests/kernels/test_cutlass.py`.
4 """
5 -from typing import Type
5 +from typing import Optional, Type
6
7 import pytest
8 import torch
@@ -27,12 +27,27 @@ def to_int8(tensor: torch.Tensor):
27 return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8)
28
29
30 +def baseline_scaled_mm(a: torch.Tensor,
31 + b: torch.Tensor,
32 + scale_a: torch.Tensor,
33 + scale_b: torch.Tensor,
34 + out_dtype: Type[torch.dtype],
35 + bias: Optional[torch.Tensor] = None) -> torch.Tensor:
36 +
37 + output = (scale_a * (scale_b * (torch.mm(
38 + a.to(dtype=torch.float32), b.to(dtype=torch.float32))))).to(out_dtype)
39 + if bias is not None:
40 + output = output + bias
41 +
42 + return output
43 +
44 +
45 def cutlass_fp8_gemm_helper(m: int,
46 n: int,
47 k: int,
48 per_token_act_quant: bool,
49 per_out_channel_weight_quant: bool,
35 - bias: bool,
50 + use_bias: bool,
51 out_dtype: Type[torch.dtype] = torch.bfloat16,
52 device: str = "cuda"):
53 # Test for a cutlass kernel with per-token activation quantization
@@ -43,23 +58,19 @@ def cutlass_fp8_gemm_helper(m: int,
58 m_a_scales = m if per_token_act_quant else 1
59 n_b_scales = n if per_out_channel_weight_quant else 1
60
46 - scale_a = (torch.randn(
47 - (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
48 - scale_b = (torch.randn(
49 - (1, n_b_scales), device=device, dtype=torch.float32) / 10)
50 - if bias:
51 - # bias term should be > 1 so that the absolute tolerance can catch it
52 - bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
53 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
61 + scale_a = (torch.randn((m_a_scales, 1), device=device,
62 + dtype=torch.float32))
63 + scale_b = (torch.randn((1, n_b_scales), device=device,
64 + dtype=torch.float32))
65 + if use_bias:
66 + bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
67 else:
55 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
56 - bias_t = 0
68 + bias = None
69
58 - baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
59 - scale_b * b.to(dtype=torch.float32)) +
60 - bias_t).to(out_dtype)
70 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
71 + baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
72
62 - assert torch.allclose(out, baseline, rtol=1e-2, atol=1e-1)
73 + assert torch.allclose(out, baseline, rtol=1e-2, atol=5e-2)
74
75
76 def cutlass_int8_gemm_helper(m: int,
@@ -67,7 +78,7 @@ def cutlass_int8_gemm_helper(m: int,
78 k: int,
79 per_token_act_quant: bool,
80 per_out_channel_weight_quant: bool,
70 - bias: bool,
81 + use_bias: bool,
82 out_dtype: Type[torch.dtype] = torch.bfloat16,
83 device: str = "cuda"):
84 # Test for a cutlass kernel with per-token activation quantization
@@ -78,22 +89,19 @@ def cutlass_int8_gemm_helper(m: int,
89 m_a_scales = m if per_token_act_quant else 1
90 n_b_scales = n if per_out_channel_weight_quant else 1
91
81 - scale_a = (torch.randn(
82 - (m_a_scales, 1), device=device, dtype=torch.float32) / 10)
83 - scale_b = (torch.randn(
84 - (1, n_b_scales), device=device, dtype=torch.float32) / 10)
92 + scale_a = (torch.randn((m_a_scales, 1), device=device,
93 + dtype=torch.float32))
94 + scale_b = (torch.randn((1, n_b_scales), device=device,
95 + dtype=torch.float32))
96
86 - if bias:
87 - # bias term should be > 1 so that the absolute tolerance can catch it
88 - bias_t = torch.rand((n, ), device=device, dtype=out_dtype) + 1.0
89 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias_t)
97 + if use_bias:
98 + bias = torch.rand((n, ), device=device, dtype=out_dtype) * 10
99 else:
91 - out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype)
92 - bias_t = 0
100 + bias = None
101 +
102 + out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
103 + baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
104
94 - baseline = (torch.mm(scale_a * a.to(dtype=torch.float32),
95 - scale_b * b.to(dtype=torch.float32)) +
96 - bias_t).to(dtype=out_dtype)
105 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
106
107
@@ -102,12 +110,12 @@ def cutlass_int8_gemm_helper(m: int,
110 @pytest.mark.parametrize("k", [128, 496, 1024])
111 @pytest.mark.parametrize("per_act_token", [True, False])
112 @pytest.mark.parametrize("per_out_ch", [True, False])
105 -@pytest.mark.parametrize("bias", [True, False])
113 +@pytest.mark.parametrize("use_bias", [True, False])
114 @pytest.mark.skipif(capability < 89,
115 reason="FP8 is not supported on this GPU type.")
116 def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
109 - per_out_ch: bool, bias: bool):
110 - cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
117 + per_out_ch: bool, use_bias: bool):
118 + cutlass_fp8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
119
120
121 @pytest.mark.parametrize("m", [512, 222, 33, 1])
@@ -115,70 +123,70 @@ def test_cutlass_fp8_gemm(m: int, n: int, k: int, per_act_token: bool,
123 @pytest.mark.parametrize("k", [128, 496, 1024])
124 @pytest.mark.parametrize("per_act_token", [True, False])
125 @pytest.mark.parametrize("per_out_ch", [True, False])
118 -@pytest.mark.parametrize("bias", [True, False])
126 +@pytest.mark.parametrize("use_bias", [True, False])
127 def test_cutlass_int8_gemm(m: int, n: int, k: int, per_act_token: bool,
120 - per_out_ch: bool, bias: bool):
121 - cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, bias)
128 + per_out_ch: bool, use_bias: bool):
129 + cutlass_int8_gemm_helper(m, n, k, per_act_token, per_out_ch, use_bias)
130
131
132 @pytest.mark.parametrize("per_act_token", [True, False])
133 @pytest.mark.parametrize("per_out_ch", [True, False])
134 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
127 -@pytest.mark.parametrize("bias", [True, False])
135 +@pytest.mark.parametrize("use_bias", [True, False])
136 def test_cutlass_int8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
137 out_dtype: Type[torch.dtype],
130 - bias: bool):
138 + use_bias: bool):
139 cutlass_int8_gemm_helper(512,
140 512,
141 512,
142 per_act_token,
143 per_out_ch,
136 - bias,
144 + use_bias,
145 out_dtype=out_dtype)
146
147
148 @pytest.mark.parametrize("per_act_token", [True, False])
149 @pytest.mark.parametrize("per_out_ch", [True, False])
150 @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
143 -@pytest.mark.parametrize("bias", [True, False])
151 +@pytest.mark.parametrize("use_bias", [True, False])
152 @pytest.mark.skipif(capability < 89,
153 reason="FP8 is not supported on this GPU type.")
154 def test_cutlass_fp8_gemm_output_dtype(per_act_token: bool, per_out_ch: bool,
155 out_dtype: Type[torch.dtype],
148 - bias: bool):
156 + use_bias: bool):
157 cutlass_fp8_gemm_helper(512,
158 512,
159 512,
160 per_act_token,
161 per_out_ch,
154 - bias,
162 + use_bias,
163 out_dtype=out_dtype)
164
165
166 @pytest.mark.parametrize("per_act_token", [True, False])
167 @pytest.mark.parametrize("per_out_ch", [True, False])
160 -@pytest.mark.parametrize("bias", [True, False])
168 +@pytest.mark.parametrize("use_bias", [True, False])
169 @pytest.mark.parametrize("device", CUDA_DEVICES)
170 @pytest.mark.skipif(capability < 89,
171 reason="FP8 is not supported on this GPU type.")
172 def test_cutlass_fp8_gemm_devices(per_act_token: bool, per_out_ch: bool,
165 - bias: bool, device: str):
166 - cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, bias,
173 + use_bias: bool, device: str):
174 + cutlass_fp8_gemm_helper(512, 512, 512, per_act_token, per_out_ch, use_bias,
175 torch.bfloat16, device)
176
177
178 @pytest.mark.parametrize("per_act_token", [True, False])
179 @pytest.mark.parametrize("per_out_ch", [True, False])
172 -@pytest.mark.parametrize("bias", [True, False])
180 +@pytest.mark.parametrize("use_bias", [True, False])
181 @pytest.mark.parametrize("device", CUDA_DEVICES)
182 def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
175 - bias: bool, device: str):
183 + use_bias: bool, device: str):
184 cutlass_int8_gemm_helper(512,
185 512,
186 512,
187 per_act_token,
188 per_out_ch,
181 - bias,
189 + use_bias,
190 out_dtype=torch.bfloat16,
191 device=device)
192
@@ -190,25 +198,26 @@ def test_cutlass_int8_gemm_devices(per_act_token: bool, per_out_ch: bool,
198 # kernel must handle any M thrown at it.
199 @pytest.mark.parametrize("per_act_token", [True, False])
200 @pytest.mark.parametrize("per_out_ch", [True, False])
193 -@pytest.mark.parametrize("bias", [True, False])
201 +@pytest.mark.parametrize("use_bias", [True, False])
202 @pytest.mark.skipif(capability < 89,
203 reason="FP8 is not supported on this GPU type.")
204 def test_cutlass_fp8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
197 - bias: bool):
205 + use_bias: bool):
206 for nk in range(32, 128, 32):
207 for m in range(1, 128):
200 - cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch, bias)
208 + cutlass_fp8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
209 + use_bias)
210
211
212 @pytest.mark.parametrize("per_act_token", [True, False])
213 @pytest.mark.parametrize("per_out_ch", [True, False])
205 -@pytest.mark.parametrize("bias", [True, False])
214 +@pytest.mark.parametrize("use_bias", [True, False])
215 def test_cutlass_int8_gemm_m_sweep(per_act_token: bool, per_out_ch: bool,
207 - bias: bool):
216 + use_bias: bool):
217 for nk in range(32, 128, 32):
218 for m in range(1, 128):
219 cutlass_int8_gemm_helper(m, nk, nk, per_act_token, per_out_ch,
211 - bias)
220 + use_bias)
221
222
223 # Test working with a subset of A and B
@@ -229,9 +238,11 @@ def test_cutlass_subset():
238 scale_a,
239 scale_b,
240 out_dtype=torch.bfloat16)
232 - baseline = torch.mm(scale_a * a.to(dtype=torch.float32),
233 - scale_b *
234 - b.to(dtype=torch.float32)).to(dtype=torch.bfloat16)
241 + baseline = baseline_scaled_mm(a,
242 + b,
243 + scale_a,
244 + scale_b,
245 + out_dtype=torch.bfloat16)
246
247 assert torch.allclose(out, baseline, rtol=1e-1, atol=1e0)
248