| 1 | #!/usr/bin/env python3 |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | """Tests for the ROCm Triton W4A16 GEMM kernel. |
| 5 | |
| 6 | Run `pytest tests/kernels/quantization/test_triton_w4a16.py`. |
| 7 | """ |
| 8 | |
| 9 | import importlib |
| 10 | |
| 11 | import pytest |
| 12 | import torch |
| 13 | from torch._subclasses.fake_tensor import FakeTensorMode |
| 14 | from torch.fx.experimental.proxy_tensor import make_fx |
| 15 | |
| 16 | from vllm.platforms import current_platform |
| 17 | from vllm.utils.torch_utils import set_random_seed |
| 18 | |
| 19 | # This test module is ROCm/Triton specific. Avoid import-time failures on |
| 20 | # non-ROCm or environments without Triton by skipping early. |
| 21 | if not current_platform.is_rocm(): |
| 22 | pytest.skip("ROCm only", allow_module_level=True) |
| 23 | |
| 24 | pytest.importorskip("triton") |
| 25 | |
| 26 | device = "cuda" |
| 27 | |
| 28 | triton_w4a16_module = importlib.import_module( |
| 29 | "vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16" |
| 30 | ) |
| 31 | triton_w4a16_gemm = triton_w4a16_module.triton_w4a16_gemm |
| 32 | TritonW4A16LinearKernel = triton_w4a16_module.TritonW4A16LinearKernel |
| 33 | |
| 34 | |
| 35 | def _pack_int4_along_n(w_int4_kn: torch.Tensor) -> torch.Tensor: |
| 36 | """Pack int4 values along N: [K, N] -> [K, N//8] int32.""" |
| 37 | assert w_int4_kn.dtype == torch.int32 |
| 38 | K, N = w_int4_kn.shape |
| 39 | assert N % 8 == 0 |
| 40 | shifts = torch.arange(8, device=w_int4_kn.device, dtype=torch.int32) * 4 |
| 41 | return torch.sum( |
| 42 | (w_int4_kn.view(K, N // 8, 8) & 0xF) << shifts, |
| 43 | dim=2, |
| 44 | dtype=torch.int32, |
| 45 | ).contiguous() |
| 46 | |
| 47 | |
| 48 | def _unpack_int4_along_n(w_packed_kn8: torch.Tensor) -> torch.Tensor: |
| 49 | """Unpack int4 values along N: [K, N//8] -> [K, N] int32.""" |
| 50 | assert w_packed_kn8.dtype == torch.int32 |
| 51 | K, N8 = w_packed_kn8.shape |
| 52 | shifts = torch.arange(8, device=w_packed_kn8.device, dtype=torch.int32) * 4 |
| 53 | nibbles = (w_packed_kn8.unsqueeze(-1) >> shifts) & 0xF |
| 54 | return nibbles.reshape(K, N8 * 8) |
| 55 | |
| 56 | |
| 57 | def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor: |
| 58 | """Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8].""" |
| 59 | assert w_int4_kn.dtype == torch.int32 |
| 60 | K, N = w_int4_kn.shape |
| 61 | assert K % 8 == 0 |
| 62 | out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device) |
| 63 | for i in range(8): |
| 64 | out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4) |
| 65 | return out.contiguous() |
| 66 | |
| 67 | |
| 68 | def _w4a16_reference( |
| 69 | a_mk: torch.Tensor, |
| 70 | b_packed_kn8: torch.Tensor, |
| 71 | scales_gn: torch.Tensor, |
| 72 | *, |
| 73 | group_size: int, |
| 74 | qzeros_gn8: torch.Tensor | None, |
| 75 | zp_bias: int, |
| 76 | ) -> torch.Tensor: |
| 77 | """Reference implementation for W4A16. |
| 78 | |
| 79 | a_mk: [M,K] fp16/bf16 |
| 80 | b_packed_kn8: [K, N//8] int32, N-packed int4 weights |
| 81 | scales_gn: [K//G, N] fp16/bf16 |
| 82 | qzeros_gn8: [K//G, N//8] int32, N-packed int4 zeros, or None |
| 83 | """ |
| 84 | assert a_mk.dtype in (torch.float16, torch.bfloat16) |
| 85 | assert b_packed_kn8.dtype == torch.int32 |
| 86 | assert scales_gn.dtype == a_mk.dtype |
| 87 | |
| 88 | M, K = a_mk.shape |
| 89 | N = b_packed_kn8.shape[1] * 8 |
| 90 | assert b_packed_kn8.shape[0] == K |
| 91 | |
| 92 | assert group_size > 0 and K % group_size == 0 |
| 93 | G = group_size |
| 94 | num_groups = K // G |
| 95 | assert scales_gn.shape == (num_groups, N) |
| 96 | |
| 97 | w_int4 = _unpack_int4_along_n(b_packed_kn8) # [K,N] |
| 98 | if qzeros_gn8 is None: |
| 99 | z_full = torch.full((K, N), zp_bias, dtype=torch.int32, device=a_mk.device) |
| 100 | else: |
| 101 | assert qzeros_gn8.shape == (num_groups, N // 8) |
| 102 | z_gn = _unpack_int4_along_n(qzeros_gn8) # [G,N] in groups |
| 103 | z_full = z_gn.repeat_interleave(G, dim=0) # [K,N] |
| 104 | |
| 105 | s_full = scales_gn.repeat_interleave(G, dim=0).to(torch.float32) # [K,N] |
| 106 | w_fp = (w_int4 - z_full).to(torch.float32) * s_full # [K,N] |
| 107 | |
| 108 | out = a_mk.to(torch.float32) @ w_fp # [M,N] |
| 109 | return out.to(a_mk.dtype) |
| 110 | |
| 111 | |
| 112 | def test_triton_w4a16_gemm_fake_impl(): |
| 113 | with FakeTensorMode(): |
| 114 | a = torch.empty((4, 256), device=device, dtype=torch.float16) |
| 115 | b_q = torch.empty((256, 64), device=device, dtype=torch.int32) |
| 116 | scales = torch.empty((2, 512), device=device, dtype=torch.float16) |
| 117 | |
| 118 | out = triton_w4a16_gemm( |
| 119 | a=a, |
| 120 | b_q=b_q, |
| 121 | scales=scales, |
| 122 | qzeros=None, |
| 123 | group_size=128, |
| 124 | ) |
| 125 | |
| 126 | assert out.shape == (4, 512) |
| 127 | assert out.dtype == a.dtype |
| 128 | assert out.device == a.device |
| 129 | |
| 130 | graph = make_fx( |
| 131 | lambda a, b_q, scales: triton_w4a16_gemm( |
| 132 | a, b_q, scales, qzeros=None, group_size=128 |
| 133 | ) |
| 134 | )(a, b_q, scales) |
| 135 | |
| 136 | assert any( |
| 137 | node.op == "call_function" |
| 138 | and node.target == torch.ops.vllm.triton_w4a16_gemm.default |
| 139 | for node in graph.graph.nodes |
| 140 | ) |
| 141 | |
| 142 | |
| 143 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") |
| 144 | @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) |
| 145 | @pytest.mark.parametrize( |
| 146 | "M,K,N,G,has_zp", |
| 147 | [ |
| 148 | (1, 256, 256, 32, False), |
| 149 | (17, 256, 512, 32, False), |
| 150 | (32, 512, 256, 64, False), |
| 151 | (33, 512, 512, 128, False), |
| 152 | (64, 1024, 256, 256, False), |
| 153 | (128, 256, 1024, 32, True), |
| 154 | (64, 512, 512, 64, True), |
| 155 | ], |
| 156 | ) |
| 157 | def test_triton_w4a16_gemm_matches_reference(dtype, M, K, N, G, has_zp): |
| 158 | if not torch.cuda.is_available(): |
| 159 | pytest.skip("CUDA/HIP device not available") |
| 160 | if N % 8 != 0 or K % G != 0: |
| 161 | pytest.skip("Invalid test shape") |
| 162 | |
| 163 | set_random_seed(0) |
| 164 | |
| 165 | a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype) |
| 166 | w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) |
| 167 | b_packed = _pack_int4_along_n(w_int4) |
| 168 | |
| 169 | scales = (0.05 * torch.rand((K // G, N), device=device, dtype=torch.float32)).to( |
| 170 | dtype |
| 171 | ) |
| 172 | |
| 173 | qzeros = None |
| 174 | if has_zp: |
| 175 | zeros_int4 = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32) |
| 176 | qzeros = _pack_int4_along_n(zeros_int4) |
| 177 | |
| 178 | out = triton_w4a16_gemm( |
| 179 | a=a, |
| 180 | b_q=b_packed, |
| 181 | scales=scales, |
| 182 | qzeros=qzeros, |
| 183 | group_size=G, |
| 184 | zp_bias=8, |
| 185 | ) |
| 186 | ref = _w4a16_reference( |
| 187 | a, |
| 188 | b_packed, |
| 189 | scales, |
| 190 | group_size=G, |
| 191 | qzeros_gn8=qzeros, |
| 192 | zp_bias=8, |
| 193 | ) |
| 194 | |
| 195 | torch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2) |
| 196 | |
| 197 | |
| 198 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") |
| 199 | def test_triton_w4a16_gemm_requires_contiguous_inputs(): |
| 200 | if not torch.cuda.is_available(): |
| 201 | pytest.skip("CUDA/HIP device not available") |
| 202 | |
| 203 | set_random_seed(0) |
| 204 | M, K, N, G = 32, 256, 256, 32 |
| 205 | a = torch.randn((K, M), device=device, dtype=torch.float16).t() # non-contiguous |
| 206 | w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) |
| 207 | b_packed = _pack_int4_along_n(w_int4) |
| 208 | scales = torch.rand((K // G, N), device=device, dtype=torch.float16) |
| 209 | |
| 210 | with pytest.raises(AssertionError): |
| 211 | triton_w4a16_gemm( |
| 212 | a=a, |
| 213 | b_q=b_packed, |
| 214 | scales=scales, |
| 215 | qzeros=None, |
| 216 | group_size=G, |
| 217 | zp_bias=8, |
| 218 | ) |
| 219 | |
| 220 | |
| 221 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") |
| 222 | def test_triton_w4a16_process_weights_after_loading_repacks_layout(): |
| 223 | if not torch.cuda.is_available(): |
| 224 | pytest.skip("CUDA/HIP device not available") |
| 225 | |
| 226 | from vllm.config import VllmConfig, set_current_vllm_config |
| 227 | from vllm.distributed import ( |
| 228 | ensure_model_parallel_initialized, |
| 229 | init_distributed_environment, |
| 230 | ) |
| 231 | from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( |
| 232 | MPLinearLayerConfig, |
| 233 | ) |
| 234 | from vllm.model_executor.parameter import ( |
| 235 | GroupQuantScaleParameter, |
| 236 | PackedColumnParameter, |
| 237 | PackedvLLMParameter, |
| 238 | ) |
| 239 | from vllm.scalar_type import scalar_types |
| 240 | |
| 241 | with set_current_vllm_config(VllmConfig()): |
| 242 | init_distributed_environment( |
| 243 | world_size=1, |
| 244 | rank=0, |
| 245 | distributed_init_method="tcp://127.0.0.1:0", |
| 246 | local_rank=0, |
| 247 | ) |
| 248 | ensure_model_parallel_initialized(1, 1) |
| 249 | |
| 250 | set_random_seed(0) |
| 251 | |
| 252 | # Small-but-nontrivial shapes. |
| 253 | K, N = 256, 256 |
| 254 | G = 32 |
| 255 | assert K % 8 == 0 and N % 8 == 0 and K % G == 0 |
| 256 | |
| 257 | # Build a canonical int4 weight grid then pack into the CT checkpoint layout. |
| 258 | w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) |
| 259 | w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn) # [N, K//8] |
| 260 | |
| 261 | # Scales in CT checkpoint layout for WNA16: [N, K//G] |
| 262 | scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16) |
| 263 | |
| 264 | # Asymmetric case: zero points in CT checkpoint layout [N//8, K//G] (N-packed) |
| 265 | zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32) |
| 266 | zeros_packed_gn8 = _pack_int4_along_n(zeros_int4_gn) # [K//G, N//8] |
| 267 | zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G] |
| 268 | |
| 269 | config = MPLinearLayerConfig( |
| 270 | full_weight_shape=(K, N), |
| 271 | partition_weight_shape=(K, N), |
| 272 | weight_type=scalar_types.uint4, # asymmetric |
| 273 | act_type=torch.float16, |
| 274 | group_size=G, |
| 275 | zero_points=True, |
| 276 | ) |
| 277 | kernel = TritonW4A16LinearKernel( |
| 278 | config, |
| 279 | w_q_param_name="weight_packed", |
| 280 | w_s_param_name="weight_scale", |
| 281 | w_zp_param_name="weight_zero_point", |
| 282 | ) |
| 283 | |
| 284 | # Build dummy layer with vLLM parameter wrappers. |
| 285 | weight_loader = lambda *args, **kwargs: None |
| 286 | |
| 287 | class DummyLayer(torch.nn.Module): |
| 288 | pass |
| 289 | |
| 290 | layer = DummyLayer() |
| 291 | layer.register_parameter( |
| 292 | "weight_packed", |
| 293 | PackedvLLMParameter( |
| 294 | data=w_ckpt_nk8, |
| 295 | weight_loader=weight_loader, |
| 296 | input_dim=1, |
| 297 | output_dim=0, |
| 298 | packed_factor=8, |
| 299 | packed_dim=1, |
| 300 | ), |
| 301 | ) |
| 302 | layer.register_parameter( |
| 303 | "weight_scale", |
| 304 | GroupQuantScaleParameter( |
| 305 | data=scales_ckpt_nkg, |
| 306 | weight_loader=weight_loader, |
| 307 | input_dim=1, |
| 308 | output_dim=0, |
| 309 | ), |
| 310 | ) |
| 311 | layer.register_parameter( |
| 312 | "weight_zero_point", |
| 313 | PackedColumnParameter( |
| 314 | data=zeros_ckpt_n8kg, |
| 315 | weight_loader=weight_loader, |
| 316 | output_dim=0, |
| 317 | packed_factor=8, |
| 318 | packed_dim=0, |
| 319 | ), |
| 320 | ) |
| 321 | |
| 322 | kernel.process_weights_after_loading(layer) |
| 323 | |
| 324 | # Expected transformed layouts. |
| 325 | expected_w_kn8 = _pack_int4_along_n(w_int4_kn) # [K, N//8] |
| 326 | expected_scales_gn = scales_ckpt_nkg.t().contiguous() # [K//G, N] |
| 327 | expected_zeros_gn8 = zeros_ckpt_n8kg.t().contiguous() # [K//G, N//8] |
| 328 | |
| 329 | assert tuple(layer.weight_packed.shape) == (K, N // 8) |
| 330 | assert tuple(layer.weight_scale.shape) == (K // G, N) |
| 331 | assert tuple(layer.weight_zero_point.shape) == (K // G, N // 8) |
| 332 | |
| 333 | torch.testing.assert_close(layer.weight_packed, expected_w_kn8) |
| 334 | torch.testing.assert_close(layer.weight_scale, expected_scales_gn) |
| 335 | torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8) |
| 336 | |
| 337 | |
| 338 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") |
| 339 | def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout(): |
| 340 | """AutoGPTQ qzeros are already [K//G, N//8] (output_dim=1): no transpose.""" |
| 341 | if not torch.cuda.is_available(): |
| 342 | pytest.skip("CUDA/HIP device not available") |
| 343 | |
| 344 | from vllm.config import VllmConfig, set_current_vllm_config |
| 345 | from vllm.distributed import ( |
| 346 | ensure_model_parallel_initialized, |
| 347 | init_distributed_environment, |
| 348 | ) |
| 349 | from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( |
| 350 | MPLinearLayerConfig, |
| 351 | ) |
| 352 | from vllm.model_executor.parameter import ( |
| 353 | GroupQuantScaleParameter, |
| 354 | PackedvLLMParameter, |
| 355 | ) |
| 356 | from vllm.scalar_type import scalar_types |
| 357 | |
| 358 | with set_current_vllm_config(VllmConfig()): |
| 359 | init_distributed_environment( |
| 360 | world_size=1, |
| 361 | rank=0, |
| 362 | distributed_init_method="tcp://127.0.0.1:0", |
| 363 | local_rank=0, |
| 364 | ) |
| 365 | ensure_model_parallel_initialized(1, 1) |
| 366 | |
| 367 | set_random_seed(0) |
| 368 | |
| 369 | K, N = 256, 256 |
| 370 | G = 32 |
| 371 | |
| 372 | w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32) |
| 373 | w_gptq_k8n = _pack_int4_along_k_to_ckpt(w_int4_kn).t().contiguous() |
| 374 | scales_gptq_gn = 0.05 * torch.rand((K // G, N), device=device, dtype=torch.float16) |
| 375 | zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32) |
| 376 | zeros_gptq_gn8 = _pack_int4_along_n(zeros_int4_gn) |
| 377 | |
| 378 | config = MPLinearLayerConfig( |
| 379 | full_weight_shape=(K, N), |
| 380 | partition_weight_shape=(K, N), |
| 381 | weight_type=scalar_types.uint4, |
| 382 | act_type=torch.float16, |
| 383 | group_size=G, |
| 384 | zero_points=True, |
| 385 | ) |
| 386 | kernel = TritonW4A16LinearKernel( |
| 387 | config, |
| 388 | w_q_param_name="qweight", |
| 389 | w_s_param_name="scales", |
| 390 | w_zp_param_name="qzeros", |
| 391 | ) |
| 392 | |
| 393 | weight_loader = lambda *args, **kwargs: None |
| 394 | |
| 395 | class DummyLayer(torch.nn.Module): |
| 396 | pass |
| 397 | |
| 398 | layer = DummyLayer() |
| 399 | layer.register_parameter( |
| 400 | "qweight", |
| 401 | PackedvLLMParameter( |
| 402 | data=w_gptq_k8n, |
| 403 | weight_loader=weight_loader, |
| 404 | input_dim=0, |
| 405 | output_dim=1, |
| 406 | packed_factor=8, |
| 407 | packed_dim=0, |
| 408 | ), |
| 409 | ) |
| 410 | layer.register_parameter( |
| 411 | "scales", |
| 412 | GroupQuantScaleParameter( |
| 413 | data=scales_gptq_gn, |
| 414 | weight_loader=weight_loader, |
| 415 | input_dim=0, |
| 416 | output_dim=1, |
| 417 | ), |
| 418 | ) |
| 419 | layer.register_parameter( |
| 420 | "qzeros", |
| 421 | PackedvLLMParameter( |
| 422 | data=zeros_gptq_gn8, |
| 423 | weight_loader=weight_loader, |
| 424 | input_dim=0, |
| 425 | output_dim=1, |
| 426 | packed_factor=8, |
| 427 | packed_dim=1, |
| 428 | ), |
| 429 | ) |
| 430 | |
| 431 | kernel.process_weights_after_loading(layer) |
| 432 | |
| 433 | expected_w_kn8 = _pack_int4_along_n(w_int4_kn) |
| 434 | |
| 435 | assert tuple(layer.qweight.shape) == (K, N // 8) |
| 436 | assert tuple(layer.scales.shape) == (K // G, N) |
| 437 | assert tuple(layer.qzeros.shape) == (K // G, N // 8) |
| 438 | |
| 439 | torch.testing.assert_close(layer.qweight, expected_w_kn8) |
| 440 | torch.testing.assert_close(layer.scales, scales_gptq_gn) |
| 441 | torch.testing.assert_close(layer.qzeros, zeros_gptq_gn8) |
| 442 | |
| 443 | |
| 444 | @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only") |
| 445 | def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch): |
| 446 | """For symmetric (uint4b8) layers, apply_weights must pass qzeros=None.""" |
| 447 | from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( |
| 448 | MPLinearLayerConfig, |
| 449 | ) |
| 450 | from vllm.scalar_type import scalar_types |
| 451 | |
| 452 | K, N, G = 256, 256, 32 |
| 453 | config = MPLinearLayerConfig( |
| 454 | full_weight_shape=(K, N), |
| 455 | partition_weight_shape=(K, N), |
| 456 | weight_type=scalar_types.uint4b8, |
| 457 | act_type=torch.float16, |
| 458 | group_size=G, |
| 459 | zero_points=False, |
| 460 | ) |
| 461 | kernel = TritonW4A16LinearKernel( |
| 462 | config, |
| 463 | w_q_param_name="qweight", |
| 464 | w_s_param_name="scales", |
| 465 | w_zp_param_name="qzeros", |
| 466 | ) |
| 467 | |
| 468 | class DummyLayer(torch.nn.Module): |
| 469 | pass |
| 470 | |
| 471 | layer = DummyLayer() |
| 472 | layer.qweight = torch.empty((K, N // 8), device=device, dtype=torch.int32) |
| 473 | layer.scales = torch.empty((K // G, N), device=device, dtype=torch.float16) |
| 474 | layer.qzeros = torch.empty((1, 1), device=device, dtype=torch.int32) |
| 475 | |
| 476 | captured = {} |
| 477 | |
| 478 | def fake_gemm(*, a, b_q, scales, qzeros, group_size, zp_bias): |
| 479 | captured["qzeros"] = qzeros |
| 480 | captured["zp_bias"] = zp_bias |
| 481 | return torch.empty((a.shape[0], N), device=a.device, dtype=a.dtype) |
| 482 | |
| 483 | monkeypatch.setattr(triton_w4a16_module, "triton_w4a16_gemm", fake_gemm) |
| 484 | |
| 485 | x = torch.empty((2, K), device=device, dtype=torch.float16) |
| 486 | output = kernel.apply_weights(layer, x) |
| 487 | |
| 488 | assert output.shape == (2, N) |
| 489 | assert captured["qzeros"] is None |
| 490 | assert captured["zp_bias"] == scalar_types.uint4b8.bias |