| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import math |
| 5 | |
| 6 | import pytest |
| 7 | import torch |
| 8 | from compressed_tensors.transform import deterministic_hadamard_matrix |
| 9 | |
| 10 | from vllm import _custom_ops as ops |
| 11 | from vllm.platforms import current_platform |
| 12 | |
| 13 | if current_platform.is_rocm(): |
| 14 | pytest.skip( |
| 15 | "These tests require hadacore_transform, not supported on ROCm.", |
| 16 | allow_module_level=True, |
| 17 | ) |
| 18 | |
| 19 | |
| 20 | @pytest.mark.parametrize("batch_size", [1, 32]) |
| 21 | @pytest.mark.parametrize("hidden_dim", [2**n for n in range(10)]) |
| 22 | def test_hadacore(batch_size, hidden_dim, dtype=torch.bfloat16, device="cuda"): |
| 23 | x = torch.eye(hidden_dim, dtype=dtype, device=device) |
| 24 | hadamard = deterministic_hadamard_matrix( |
| 25 | hidden_dim, dtype=torch.float64, device="cuda" |
| 26 | ) / math.sqrt(hidden_dim) |
| 27 | |
| 28 | y = ops.hadacore_transform(x.clone()) |
| 29 | y_true = (x.to(hadamard.dtype) @ hadamard.T).to(y.dtype) |
| 30 | assert torch.allclose(y, y_true) |
| 31 | |
| 32 | y = ops.hadacore_transform(y) |
| 33 | assert torch.allclose(y, x) |