| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for cutlass kernels. |
| 4 | |
| 5 | Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`. |
| 6 | """ |
| 7 | |
| 8 | import random |
| 9 | |
| 10 | import pytest |
| 11 | import torch |
| 12 | |
| 13 | from tests.kernels.utils import baseline_scaled_mm, opcheck, to_fp8, to_int8 |
| 14 | from vllm import _custom_ops as ops |
| 15 | from vllm.platforms import current_platform |
| 16 | from vllm.utils.math_utils import cdiv |
| 17 | |
| 18 | if not current_platform.is_cuda(): |
| 19 | pytest.skip("These tests use CUTLASS which requires CUDA", allow_module_level=True) |
| 20 | |
| 21 | MNK_FACTORS = [ |
| 22 | (1, 256, 128), |
| 23 | (1, 16384, 1024), |
| 24 | (1, 24576, 496), |
| 25 | (16, 256, 496), |
| 26 | (16, 16384, 128), |
| 27 | (16, 24576, 4096), |
| 28 | (32, 8192, 4096), |
| 29 | (32, 16384, 4096), |
| 30 | (33, 1024, 1024), |
| 31 | (33, 8192, 128), |
| 32 | (64, 2048, 496), |
| 33 | (64, 16384, 1024), |
| 34 | (100, 8192, 496), |
| 35 | (128, 32768, 4096), |
| 36 | (256, 4096, 4096), |
| 37 | (512, 256, 1024), |
| 38 | (512, 8192, 4096), |
| 39 | (512, 16384, 128), |
| 40 | (512, 24576, 128), |
| 41 | ] |
| 42 | |
| 43 | # Shapes with N or K not divisible by 16. These exercise the padding path |
| 44 | # inside CutlassFP8ScaledMMLinearKernel.apply_scaled_mm (e.g. Qwen2.5-VL |
| 45 | # vision MLP dims). |
| 46 | UNALIGNED_MNK_FACTORS = [ |
| 47 | (32, 3420, 1280), |
| 48 | (32, 1280, 6840), |
| 49 | (1, 3420, 1280), |
| 50 | (64, 6840, 1280), |
| 51 | (16, 100, 200), |
| 52 | (33, 255, 513), |
| 53 | ] |
| 54 | |
| 55 | CUDA_DEVICES = [ |
| 56 | f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2) |
| 57 | ] |
| 58 | |
| 59 | # -1 means full extent in that dimension |
| 60 | TENSORWISE_GROUP_SHAPE = (-1, -1) |
| 61 | PER_TOKEN_GROUP_SHAPE = (1, -1) |
| 62 | PER_OUT_CH_GROUP_SHAPE = (-1, 1) |
| 63 | |
| 64 | capability = current_platform.get_device_capability() |
| 65 | # Fall back to 0 so import doesn't fail on unknown capability |
| 66 | capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0 |
| 67 | |
| 68 | |
| 69 | def rand_int8(shape: tuple, device: str = "cuda"): |
| 70 | return to_int8(torch.rand(shape, device=device) * 255 - 128) |
| 71 | |
| 72 | |
| 73 | def group_scale_helper(shape, group_shape): |
| 74 | return [shape[i] if s < 0 else s for i, s in enumerate(group_shape)] |
| 75 | |
| 76 | |
| 77 | def scale_shape(shape, group_shape): |
| 78 | assert len(shape) == len(group_shape) |
| 79 | group_shape = group_scale_helper(shape, group_shape) |
| 80 | return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) |
| 81 | |
| 82 | |
| 83 | def cutlass_fp8_gemm_helper( |
| 84 | m: int, |
| 85 | n: int, |
| 86 | k: int, |
| 87 | a_scale_group_shape: tuple, |
| 88 | b_scale_group_shape: tuple, |
| 89 | use_bias: bool, |
| 90 | out_dtype: type[torch.dtype] = torch.bfloat16, |
| 91 | device: str = "cuda", |
| 92 | ): |
| 93 | # Test for a cutlass kernel with per-token activation quantization |
| 94 | # and per-output channel weight quantization. |
| 95 | a = to_fp8(torch.randn((m, k), device=device)) |
| 96 | b = to_fp8(torch.randn((n, k), device=device).t()) |
| 97 | |
| 98 | a_scales_shape = scale_shape(a.shape, a_scale_group_shape) |
| 99 | b_scales_shape = scale_shape(b.shape, b_scale_group_shape) |
| 100 | |
| 101 | scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32) |
| 102 | scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32) |
| 103 | |
| 104 | # make scales M-major for blockwise quant, doesn't affect 1D scales |
| 105 | scale_a = scale_a.t().contiguous().t() |
| 106 | # make scales K-major for blockwise quant, doesn't affect 1D scales |
| 107 | scale_b = scale_b.t().contiguous().t() |
| 108 | |
| 109 | bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None |
| 110 | |
| 111 | out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) |
| 112 | baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) |
| 113 | |
| 114 | torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1) |
| 115 | |
| 116 | opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias)) |
| 117 | |
| 118 | |
| 119 | def cutlass_int8_gemm_helper( |
| 120 | m: int, |
| 121 | n: int, |
| 122 | k: int, |
| 123 | a_scale_group_shape: tuple, |
| 124 | b_scale_group_shape: tuple, |
| 125 | use_bias: bool, |
| 126 | out_dtype: type[torch.dtype] = torch.bfloat16, |
| 127 | device: str = "cuda", |
| 128 | ): |
| 129 | # Test for a cutlass kernel with per-token activation quantization |
| 130 | # and per-output channel weight quantization. |
| 131 | a = to_int8(torch.randn((m, k), device=device) * 5) |
| 132 | b = to_int8(torch.randn((n, k), device=device).t() * 5) |
| 133 | |
| 134 | a_scales_shape = scale_shape(a.shape, a_scale_group_shape) |
| 135 | b_scales_shape = scale_shape(b.shape, b_scale_group_shape) |
| 136 | |
| 137 | scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32) |
| 138 | scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32) |
| 139 | |
| 140 | bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None |
| 141 | |
| 142 | out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) |
| 143 | baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) |
| 144 | |
| 145 | torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0) |
| 146 | |
| 147 | opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias)) |
| 148 | |
| 149 | |
| 150 | @pytest.mark.parametrize("m,n,k", MNK_FACTORS) |
| 151 | @pytest.mark.parametrize( |
| 152 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 153 | ) |
| 154 | @pytest.mark.parametrize( |
| 155 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 156 | ) |
| 157 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 158 | @pytest.mark.skipif( |
| 159 | not current_platform.has_device_capability(89), |
| 160 | reason="FP8 is not supported on this GPU type.", |
| 161 | ) |
| 162 | def test_cutlass_fp8_gemm( |
| 163 | m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 164 | ): |
| 165 | cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias) |
| 166 | |
| 167 | |
| 168 | @pytest.mark.parametrize("m,n,k", UNALIGNED_MNK_FACTORS) |
| 169 | @pytest.mark.parametrize( |
| 170 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 171 | ) |
| 172 | @pytest.mark.parametrize( |
| 173 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 174 | ) |
| 175 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 176 | @pytest.mark.skipif( |
| 177 | not current_platform.has_device_capability(89), |
| 178 | reason="FP8 is not supported on this GPU type.", |
| 179 | ) |
| 180 | def test_cutlass_fp8_gemm_padded( |
| 181 | m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 182 | ): |
| 183 | """Test CUTLASS FP8 GEMM with padding for non-16-aligned N/K dims. |
| 184 | |
| 185 | Exercises CutlassFP8ScaledMMLinearKernel.apply_scaled_mm which pads |
| 186 | inputs to satisfy CUTLASS alignment requirements — the path taken by |
| 187 | models like Qwen2.5-VL whose vision MLP has non-16-aligned dims. |
| 188 | """ |
| 189 | from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( |
| 190 | CutlassFP8ScaledMMLinearKernel, |
| 191 | ) |
| 192 | |
| 193 | a = to_fp8(torch.randn((m, k), device="cuda")) |
| 194 | b = to_fp8(torch.randn((n, k), device="cuda").t()) |
| 195 | |
| 196 | a_scales_shape = scale_shape(a.shape, a_scale_group_shape) |
| 197 | b_scales_shape = scale_shape(b.shape, b_scale_group_shape) |
| 198 | |
| 199 | scale_a = torch.randn(a_scales_shape, device="cuda", dtype=torch.float32) |
| 200 | scale_b = torch.randn(b_scales_shape, device="cuda", dtype=torch.float32) |
| 201 | |
| 202 | scale_a = scale_a.t().contiguous().t() |
| 203 | scale_b = scale_b.t().contiguous().t() |
| 204 | |
| 205 | out_dtype = torch.bfloat16 |
| 206 | bias = torch.rand((n,), device="cuda", dtype=out_dtype) * 10 if use_bias else None |
| 207 | |
| 208 | baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) |
| 209 | |
| 210 | # process_weights_after_loading pad b to 16 |
| 211 | pad_k = (16 - k % 16) % 16 |
| 212 | pad_n = (16 - n % 16) % 16 |
| 213 | if pad_k > 0 or pad_n > 0: |
| 214 | b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t() |
| 215 | if pad_n > 0 and scale_b.numel() > 1: |
| 216 | scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0) |
| 217 | |
| 218 | kernel = object.__new__(CutlassFP8ScaledMMLinearKernel) |
| 219 | kernel.logical_output_size = n |
| 220 | out = kernel.apply_scaled_mm( |
| 221 | A=a, |
| 222 | B=b, |
| 223 | out_dtype=out_dtype, |
| 224 | As=scale_a, |
| 225 | Bs=scale_b, |
| 226 | bias=bias, |
| 227 | output_shape=[m, n], |
| 228 | ) |
| 229 | |
| 230 | torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1) |
| 231 | |
| 232 | |
| 233 | # Two prefill-sized cases for the SM 12.x blockwise path, where the op switches |
| 234 | # the tile scheduler to a swizzled CTA order once the weight exceeds the L2 |
| 235 | # (16384x2560 = 40 MiB and 5120x5120 = 25 MiB on a 24 MiB L2 part); the odd M |
| 236 | # also covers the non-swap-AB dispatch. Elsewhere they run the default order |
| 237 | # like the rest of the list. |
| 238 | BLOCKWISE_PREFILL_FACTORS = [ |
| 239 | (8193, 16384, 2560), |
| 240 | (5120, 5120, 5120), |
| 241 | ] |
| 242 | |
| 243 | |
| 244 | @pytest.mark.parametrize("m,n,k", MNK_FACTORS + BLOCKWISE_PREFILL_FACTORS) |
| 245 | @pytest.mark.parametrize( |
| 246 | "a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))] |
| 247 | ) |
| 248 | @pytest.mark.parametrize("use_bias", [False]) |
| 249 | @pytest.mark.skipif( |
| 250 | not current_platform.has_device_capability(90), |
| 251 | reason="FP8 blockwise is not supported on this GPU type.", |
| 252 | ) |
| 253 | def test_cutlass_fp8_blockwise_scale_gemm( |
| 254 | m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 255 | ): |
| 256 | if k % b_scale_group_shape[0] != 0 or n % b_scale_group_shape[1] != 0: |
| 257 | return |
| 258 | if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0: |
| 259 | return |
| 260 | cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias) |
| 261 | |
| 262 | |
| 263 | @pytest.mark.parametrize("m,n,k", MNK_FACTORS) |
| 264 | @pytest.mark.parametrize( |
| 265 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 266 | ) |
| 267 | @pytest.mark.parametrize( |
| 268 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 269 | ) |
| 270 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 271 | def test_cutlass_int8_gemm( |
| 272 | m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 273 | ): |
| 274 | cutlass_int8_gemm_helper( |
| 275 | m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias |
| 276 | ) |
| 277 | |
| 278 | |
| 279 | @pytest.mark.parametrize( |
| 280 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 281 | ) |
| 282 | @pytest.mark.parametrize( |
| 283 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 284 | ) |
| 285 | @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) |
| 286 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 287 | def test_cutlass_int8_gemm_output_dtype( |
| 288 | a_scale_group_shape, |
| 289 | b_scale_group_shape, |
| 290 | out_dtype: type[torch.dtype], |
| 291 | use_bias: bool, |
| 292 | ): |
| 293 | cutlass_int8_gemm_helper( |
| 294 | 512, |
| 295 | 512, |
| 296 | 512, |
| 297 | a_scale_group_shape, |
| 298 | b_scale_group_shape, |
| 299 | use_bias, |
| 300 | out_dtype=out_dtype, |
| 301 | ) |
| 302 | |
| 303 | |
| 304 | @pytest.mark.parametrize( |
| 305 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 306 | ) |
| 307 | @pytest.mark.parametrize( |
| 308 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 309 | ) |
| 310 | @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) |
| 311 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 312 | @pytest.mark.skipif( |
| 313 | not current_platform.has_device_capability(89), |
| 314 | reason="FP8 is not supported on this GPU type.", |
| 315 | ) |
| 316 | def test_cutlass_fp8_gemm_output_dtype( |
| 317 | a_scale_group_shape, |
| 318 | b_scale_group_shape, |
| 319 | out_dtype: type[torch.dtype], |
| 320 | use_bias: bool, |
| 321 | ): |
| 322 | cutlass_fp8_gemm_helper( |
| 323 | 512, |
| 324 | 512, |
| 325 | 512, |
| 326 | a_scale_group_shape, |
| 327 | b_scale_group_shape, |
| 328 | use_bias, |
| 329 | out_dtype=out_dtype, |
| 330 | ) |
| 331 | |
| 332 | |
| 333 | @pytest.mark.parametrize( |
| 334 | "a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))] |
| 335 | ) |
| 336 | @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) |
| 337 | @pytest.mark.parametrize("use_bias", [False]) |
| 338 | @pytest.mark.skipif( |
| 339 | not current_platform.has_device_capability(90), |
| 340 | reason="FP8 blockwise is not supported on this GPU type.", |
| 341 | ) |
| 342 | def test_cutlass_fp8_blockwise_scale_gemm_dtype( |
| 343 | a_scale_group_shape, |
| 344 | b_scale_group_shape, |
| 345 | out_dtype: type[torch.dtype], |
| 346 | use_bias: bool, |
| 347 | ): |
| 348 | cutlass_fp8_gemm_helper( |
| 349 | 512, |
| 350 | 512, |
| 351 | 512, |
| 352 | a_scale_group_shape, |
| 353 | b_scale_group_shape, |
| 354 | use_bias, |
| 355 | out_dtype=out_dtype, |
| 356 | ) |
| 357 | |
| 358 | |
| 359 | @pytest.mark.parametrize( |
| 360 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 361 | ) |
| 362 | @pytest.mark.parametrize( |
| 363 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 364 | ) |
| 365 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 366 | @pytest.mark.parametrize("device", CUDA_DEVICES) |
| 367 | @pytest.mark.skipif( |
| 368 | not current_platform.has_device_capability(89), |
| 369 | reason="FP8 is not supported on this GPU type.", |
| 370 | ) |
| 371 | def test_cutlass_fp8_gemm_devices( |
| 372 | a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str |
| 373 | ): |
| 374 | cutlass_fp8_gemm_helper( |
| 375 | 512, |
| 376 | 512, |
| 377 | 512, |
| 378 | a_scale_group_shape, |
| 379 | b_scale_group_shape, |
| 380 | use_bias, |
| 381 | torch.bfloat16, |
| 382 | device, |
| 383 | ) |
| 384 | |
| 385 | |
| 386 | @pytest.mark.parametrize( |
| 387 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 388 | ) |
| 389 | @pytest.mark.parametrize( |
| 390 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 391 | ) |
| 392 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 393 | @pytest.mark.parametrize("device", CUDA_DEVICES) |
| 394 | def test_cutlass_int8_gemm_devices( |
| 395 | a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str |
| 396 | ): |
| 397 | cutlass_int8_gemm_helper( |
| 398 | 512, |
| 399 | 512, |
| 400 | 512, |
| 401 | a_scale_group_shape, |
| 402 | b_scale_group_shape, |
| 403 | use_bias, |
| 404 | out_dtype=torch.bfloat16, |
| 405 | device=device, |
| 406 | ) |
| 407 | |
| 408 | |
| 409 | # For the following two tests: |
| 410 | # N and K correspond to the size of the weight matrix and likely to be multiples |
| 411 | # of a large power of two. In any case, the kernel will have a naive fallback |
| 412 | # when N and K are not divisible by 16. But M is the number of tokens and the |
| 413 | # kernel must handle any M thrown at it. |
| 414 | @pytest.mark.parametrize( |
| 415 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 416 | ) |
| 417 | @pytest.mark.parametrize( |
| 418 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 419 | ) |
| 420 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 421 | @pytest.mark.skipif( |
| 422 | not current_platform.has_device_capability(89), |
| 423 | reason="FP8 is not supported on this GPU type.", |
| 424 | ) |
| 425 | def test_cutlass_fp8_gemm_m_sweep( |
| 426 | a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 427 | ): |
| 428 | for nk in range(32, 128, 32): |
| 429 | for m in range(1, 128): |
| 430 | cutlass_fp8_gemm_helper( |
| 431 | m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias |
| 432 | ) |
| 433 | |
| 434 | |
| 435 | @pytest.mark.parametrize( |
| 436 | "a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 437 | ) |
| 438 | @pytest.mark.parametrize( |
| 439 | "b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE] |
| 440 | ) |
| 441 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 442 | def test_cutlass_int8_gemm_m_sweep( |
| 443 | a_scale_group_shape, b_scale_group_shape, use_bias: bool |
| 444 | ): |
| 445 | for nk in range(32, 128, 32): |
| 446 | for m in range(1, 128): |
| 447 | cutlass_int8_gemm_helper( |
| 448 | m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias |
| 449 | ) |
| 450 | |
| 451 | |
| 452 | @pytest.mark.parametrize("m", [32, 64, 128]) |
| 453 | @pytest.mark.parametrize("n", [16, 32, 64]) |
| 454 | @pytest.mark.parametrize("k", [64, 128, 256]) |
| 455 | @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) |
| 456 | @pytest.mark.skip |
| 457 | def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int, out_dtype: torch.dtype): |
| 458 | # Currently, the test is failing because folding azp into |
| 459 | # 16-bit bias loses too much precision |
| 460 | scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10 |
| 461 | scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10 |
| 462 | |
| 463 | aq_i8 = rand_int8((m, k)) |
| 464 | bq_i8 = rand_int8((n, k)).t() |
| 465 | |
| 466 | aq_i32 = aq_i8.to(dtype=torch.int32) |
| 467 | bq_i32 = bq_i8.to(dtype=torch.int32) |
| 468 | |
| 469 | aq_f32 = aq_i8.to(dtype=torch.float32) |
| 470 | bq_f32 = bq_i8.to(dtype=torch.float32) |
| 471 | |
| 472 | b_dq = scale_b * bq_f32 |
| 473 | |
| 474 | azp_a = torch.rand((1,), device="cuda", dtype=torch.float32) * 10 + 1.5 |
| 475 | azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8) |
| 476 | azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding |
| 477 | |
| 478 | a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32) |
| 479 | torch.testing.assert_close(a_dq, scale_a * aq_f32 + azp_a) |
| 480 | |
| 481 | baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype) |
| 482 | |
| 483 | J = torch.ones((1, k), device="cuda", dtype=torch.float32) |
| 484 | azp_bias = (azp_a * scale_b * (J @ bq_f32)).to(out_dtype) |
| 485 | assert azp_bias.shape == (1, n) |
| 486 | assert azp_bias[0, :].shape == (n,) |
| 487 | |
| 488 | baseline_q = ( |
| 489 | scale_a.to(device="cpu") |
| 490 | * scale_b.to(device="cpu") |
| 491 | * ((aq_i32 + azp_aq_i8).to(device="cpu") @ bq_i32.to(device="cpu")) |
| 492 | ).to(dtype=out_dtype, device="cuda") |
| 493 | |
| 494 | out = ops.cutlass_scaled_mm( |
| 495 | aq_i8, bq_i8, scale_a, scale_b, out_dtype=out_dtype, bias=azp_bias[0, :] |
| 496 | ) |
| 497 | torch.testing.assert_close(out, baseline_dq, rtol=1e-2, atol=1e0) |
| 498 | torch.testing.assert_close(out, baseline_q, rtol=1e-2, atol=1e0) |
| 499 | |
| 500 | |
| 501 | @pytest.mark.parametrize("m", [32, 64, 128]) |
| 502 | @pytest.mark.parametrize("n", [16, 32, 64]) |
| 503 | @pytest.mark.parametrize("k", [64, 128, 256]) |
| 504 | @pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) |
| 505 | @pytest.mark.parametrize("use_bias", [True, False]) |
| 506 | @pytest.mark.parametrize("azp_per_token", [True, False]) |
| 507 | def test_cutlass_int8_azp( |
| 508 | m: int, n: int, k: int, out_dtype: torch.dtype, use_bias: bool, azp_per_token: bool |
| 509 | ): |
| 510 | m_azp = m if azp_per_token else 1 |
| 511 | scale_a = torch.randn((m_azp, 1), device="cuda", dtype=torch.float32) / 10 |
| 512 | scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10 |
| 513 | |
| 514 | aq_i8 = rand_int8((m, k)) |
| 515 | aq_i32 = aq_i8.to(dtype=torch.int32) |
| 516 | aq_f32 = aq_i8.to(dtype=torch.float32) |
| 517 | |
| 518 | bq_i8 = rand_int8((n, k)).t() |
| 519 | bq_i32 = bq_i8.to(dtype=torch.int32) |
| 520 | bq_f32 = bq_i8.to(dtype=torch.float32) |
| 521 | b_dq = scale_b * bq_f32 |
| 522 | |
| 523 | azp_a = torch.rand((m_azp, 1), device="cuda", dtype=torch.float32) * 10 + 1.5 |
| 524 | azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8) |
| 525 | azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding |
| 526 | |
| 527 | a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32) |
| 528 | torch.testing.assert_close(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3) |
| 529 | |
| 530 | if use_bias: |
| 531 | bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5 |
| 532 | else: |
| 533 | bias = torch.zeros((1, n), device="cuda", dtype=out_dtype) |
| 534 | |
| 535 | baseline_dq = (torch.mm(a_dq, b_dq) + bias).to(out_dtype) |
| 536 | |
| 537 | # int32 mm not supported on CUDA |
| 538 | a_noazp_i32_cpu = (aq_i32 - azp_aq_i8).to(device="cpu") |
| 539 | cq = (a_noazp_i32_cpu @ bq_i32.to(device="cpu")).to(device="cuda") |
| 540 | baseline_q = (scale_a * scale_b * cq + bias).to(dtype=out_dtype) |
| 541 | |
| 542 | # Hadamard is just the sum of the cols |
| 543 | azp_adj_i32 = bq_i32.sum(dim=0, keepdim=True, dtype=torch.int32) |
| 544 | azp_i32 = azp_aq_i8.to(dtype=torch.int32) |
| 545 | func_bias = bias if use_bias else None |
| 546 | |
| 547 | if azp_per_token: |
| 548 | out = ops.cutlass_scaled_mm_azp( |
| 549 | aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_adj_i32, azp_i32, func_bias |
| 550 | ) |
| 551 | else: |
| 552 | azp_with_adj_i32 = azp_i32 * azp_adj_i32 |
| 553 | out = ops.cutlass_scaled_mm_azp( |
| 554 | aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_with_adj_i32, None, func_bias |
| 555 | ) |
| 556 | |
| 557 | # bfloat16 precision is 7-bit mantissa -> 2^-8 ~ 0.4% |
| 558 | # float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05% |
| 559 | rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3 |
| 560 | atol = 1e-3 |
| 561 | torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol) |
| 562 | torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol) |
| 563 | |
| 564 | if azp_per_token: |
| 565 | opcheck( |
| 566 | torch.ops._C.cutlass_scaled_mm_azp, |
| 567 | (out, aq_i8, bq_i8, scale_a, scale_b, azp_adj_i32, azp_i32, func_bias), |
| 568 | ) |
| 569 | else: |
| 570 | opcheck( |
| 571 | torch.ops._C.cutlass_scaled_mm_azp, |
| 572 | (out, aq_i8, bq_i8, scale_a, scale_b, azp_with_adj_i32, None, func_bias), |
| 573 | ) |
| 574 | |
| 575 | |
| 576 | # Test working with a subset of A and B |
| 577 | def test_cutlass_subset(): |
| 578 | big_m, big_n, big_k = 1024, 1024, 1024 |
| 579 | m, n, k = 512, 512, 512 |
| 580 | |
| 581 | whole_a = to_int8(torch.randn((big_m, big_k), device="cuda") * 5) |
| 582 | whole_b = to_int8(torch.randn((big_n, big_k), device="cuda").t() * 5) |
| 583 | a = whole_a[0:m, 0:k] |
| 584 | b = whole_b[0:k, 0:n] |
| 585 | |
| 586 | scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10 |
| 587 | scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10 |
| 588 | |
| 589 | out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16) |
| 590 | baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16) |
| 591 | |
| 592 | torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0) |
| 593 | |
| 594 | |
| 595 | # Test to make sure cuda graphs work |
| 596 | class CutlassLayer(torch.nn.Module): |
| 597 | def __init__(self, b, scale_a, scale_b, out_dtype): |
| 598 | super().__init__() |
| 599 | self.b = b |
| 600 | self.scale_a = scale_a |
| 601 | self.scale_b = scale_b |
| 602 | self.out_dtype = out_dtype |
| 603 | |
| 604 | def forward(self, a): |
| 605 | return ops.cutlass_scaled_mm( |
| 606 | a, self.b, self.scale_a, self.scale_b, self.out_dtype |
| 607 | ) |
| 608 | |
| 609 | |
| 610 | @pytest.mark.parametrize("per_act_token", [True, False]) |
| 611 | @pytest.mark.parametrize("per_out_ch", [True, False]) |
| 612 | def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool): |
| 613 | m, n, k = 512, 512, 512 |
| 614 | |
| 615 | a = to_int8(torch.randn((m, k), device="cuda")) |
| 616 | b = to_int8(torch.randn((n, k), device="cuda").t()) |
| 617 | |
| 618 | m_a_scales = m if per_act_token else 1 |
| 619 | n_b_scales = n if per_out_ch else 1 |
| 620 | |
| 621 | scale_a = torch.randn((m_a_scales, 1), device="cuda", dtype=torch.float32) / 10 |
| 622 | scale_b = torch.randn((1, n_b_scales), device="cuda", dtype=torch.float32) / 10 |
| 623 | |
| 624 | # Construct a trivial model with a single layer that calls a CUTLASS kernel |
| 625 | model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16) |
| 626 | |
| 627 | # Run the model with a cuda graph |
| 628 | stream = torch.cuda.Stream() |
| 629 | with torch.cuda.stream(stream): |
| 630 | g = torch.cuda.CUDAGraph() |
| 631 | with torch.cuda.graph(g): |
| 632 | out = model(a) |
| 633 | out.zero_() |
| 634 | g.replay() |
| 635 | |
| 636 | baseline = torch.mm( |
| 637 | scale_a * a.to(dtype=torch.float32), scale_b * b.to(dtype=torch.float32) |
| 638 | ).to(torch.bfloat16) |
| 639 | torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0) |
| 640 | |
| 641 | |
| 642 | def test_cutlass_support_opcheck(): |
| 643 | opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,)) |
| 644 | |
| 645 | |
| 646 | @pytest.mark.parametrize("num_experts", [8, 64]) |
| 647 | @pytest.mark.parametrize("per_act_token", [True, False]) |
| 648 | @pytest.mark.parametrize("per_out_ch", [True, False]) |
| 649 | @pytest.mark.parametrize("use_bias", [False]) |
| 650 | @pytest.mark.skipif( |
| 651 | (lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))( |
| 652 | current_platform.get_device_capability() |
| 653 | ), |
| 654 | reason="Grouped gemm is not supported on this GPU type.", |
| 655 | ) |
| 656 | def test_cutlass_fp8_group_gemm( |
| 657 | num_experts: int, per_act_token: bool, per_out_ch: bool, use_bias: bool |
| 658 | ): |
| 659 | # Device and dtype setup |
| 660 | device = "cuda" |
| 661 | out_dtype = torch.half |
| 662 | |
| 663 | # Create separate A, B, C tensors for each group |
| 664 | a_tensors = [] |
| 665 | b_tensors = [] |
| 666 | a_scales_tensors = [] |
| 667 | b_scales_tensors = [] |
| 668 | baseline_tensors = [] |
| 669 | |
| 670 | expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int64) |
| 671 | |
| 672 | problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32) |
| 673 | |
| 674 | if not per_act_token: |
| 675 | one_scale_a = torch.randn((1, 1), device=device, dtype=torch.float32) |
| 676 | |
| 677 | alignment = 16 # 128 // 8 |
| 678 | # For variation, each group has dimensions |
| 679 | n_g = alignment * random.randint(1, 64) |
| 680 | k_g = alignment * random.randint(1, 64) |
| 681 | for g in range(num_experts): |
| 682 | m_g = alignment * random.randint(1, 64) |
| 683 | |
| 684 | expert_offsets[g + 1] = expert_offsets[g] + m_g |
| 685 | problem_sizes[g][0] = m_g |
| 686 | problem_sizes[g][1] = n_g |
| 687 | problem_sizes[g][2] = k_g |
| 688 | |
| 689 | m_a_scales = m_g if per_act_token else 1 |
| 690 | n_b_scales = n_g if per_out_ch else 1 |
| 691 | |
| 692 | # Create group-specific A and B (FP8) and output (FP16/FP32) |
| 693 | a_g = to_fp8(torch.randn((m_g, k_g), device=device)) |
| 694 | b_g = to_fp8(torch.randn((n_g, k_g), device=device).t()) |
| 695 | a_tensors.append(a_g) |
| 696 | b_tensors.append(b_g) |
| 697 | |
| 698 | # Set up A/B scales |
| 699 | scale_b = torch.randn((1, n_b_scales), device=device, dtype=torch.float32) |
| 700 | b_scales_tensors.append(scale_b) |
| 701 | |
| 702 | if per_act_token: |
| 703 | scale_a = torch.randn((m_a_scales, 1), device=device, dtype=torch.float32) |
| 704 | a_scales_tensors.append(scale_a) |
| 705 | else: |
| 706 | scale_a = one_scale_a |
| 707 | |
| 708 | # Compute baseline result for this group |
| 709 | baseline_g = baseline_scaled_mm(a_g, b_g, scale_a, scale_b, out_dtype, None) |
| 710 | baseline_tensors.append(baseline_g) |
| 711 | |
| 712 | a_tensors_stacked = torch.empty( |
| 713 | (expert_offsets[num_experts], k_g), device=device, dtype=torch.float8_e4m3fn |
| 714 | ) |
| 715 | b_tensors_stacked = torch.empty( |
| 716 | (num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn |
| 717 | ) |
| 718 | |
| 719 | for g in range(num_experts): |
| 720 | a_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g] |
| 721 | b_tensors_stacked[g] = b_tensors[g].t() |
| 722 | b_tensors_stacked = b_tensors_stacked.transpose(1, 2) |
| 723 | |
| 724 | if per_act_token: |
| 725 | a_scales_tensors_stacked = torch.empty( |
| 726 | (expert_offsets[num_experts], 1), device=device, dtype=torch.float32 |
| 727 | ) |
| 728 | for g in range(num_experts): |
| 729 | a_scales_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = ( |
| 730 | a_scales_tensors[g] |
| 731 | ) |
| 732 | else: |
| 733 | a_scales_tensors_stacked = one_scale_a |
| 734 | |
| 735 | b_scales_tensors_stacked = torch.empty( |
| 736 | (num_experts, n_b_scales), device=device, dtype=torch.float32 |
| 737 | ) |
| 738 | for g in range(num_experts): |
| 739 | b_scales_tensors_stacked[g] = b_scales_tensors[g] |
| 740 | |
| 741 | out_tensors_stacked = torch.zeros( |
| 742 | (expert_offsets[num_experts], n_g), device=device, dtype=out_dtype |
| 743 | ) |
| 744 | |
| 745 | ab_strides = torch.full( |
| 746 | (num_experts,), a_tensors_stacked.stride(0), device="cuda", dtype=torch.int64 |
| 747 | ) |
| 748 | c_strides = torch.full( |
| 749 | (num_experts,), out_tensors_stacked.stride(0), device="cuda", dtype=torch.int64 |
| 750 | ) |
| 751 | |
| 752 | ops.cutlass_moe_mm( |
| 753 | out_tensors_stacked, |
| 754 | a_tensors_stacked, |
| 755 | b_tensors_stacked, |
| 756 | a_scales_tensors_stacked, |
| 757 | b_scales_tensors_stacked, |
| 758 | expert_offsets[:-1], |
| 759 | problem_sizes, |
| 760 | ab_strides, |
| 761 | ab_strides, |
| 762 | c_strides, |
| 763 | per_act_token, |
| 764 | per_out_ch, |
| 765 | ) |
| 766 | |
| 767 | # Validate each group's result against the baseline |
| 768 | for g in range(num_experts): |
| 769 | baseline = baseline_tensors[g] |
| 770 | c = out_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] |
| 771 | torch.testing.assert_close(c, baseline, rtol=1e-2, atol=5e-4) |