| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Tests for the shuffle_rows function. |
| 4 | |
| 5 | Run `pytest tests/kernels/test_shuffle_rows.py`. |
| 6 | """ |
| 7 | |
| 8 | import pytest |
| 9 | import torch |
| 10 | |
| 11 | from vllm._custom_ops import shuffle_rows |
| 12 | from vllm.platforms import current_platform |
| 13 | |
| 14 | |
| 15 | @pytest.mark.parametrize("num_tokens", [1, 16, 64, 128, 256, 512, 1024]) |
| 16 | @pytest.mark.parametrize("hidden_size", [128, 256, 512, 1024, 2048, 4096]) |
| 17 | @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) |
| 18 | def test_shuffle_rows_basic(num_tokens: int, hidden_size: int, dtype: torch.dtype): |
| 19 | """Test basic functionality of shuffle_rows with various tensor sizes and |
| 20 | dtypes.""" |
| 21 | if not current_platform.is_cuda(): |
| 22 | pytest.skip("shuffle_rows requires CUDA") |
| 23 | |
| 24 | # Create input tensor |
| 25 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 26 | |
| 27 | # Create a simple permutation map (identity mapping) |
| 28 | dst2src_map = torch.arange(num_tokens, device="cuda", dtype=torch.int32) |
| 29 | |
| 30 | # Test shuffle_rows |
| 31 | output = shuffle_rows(input_tensor, dst2src_map) |
| 32 | |
| 33 | # With identity mapping, output should be identical to input |
| 34 | torch.testing.assert_close(output, input_tensor, atol=0, rtol=0) |
| 35 | |
| 36 | # Check output shape |
| 37 | assert output.shape == (num_tokens, hidden_size) |
| 38 | assert output.dtype == dtype |
| 39 | assert output.device == input_tensor.device |
| 40 | |
| 41 | |
| 42 | @pytest.mark.parametrize("num_tokens", [16, 64, 128]) |
| 43 | @pytest.mark.parametrize("hidden_size", [128, 512, 1024]) |
| 44 | @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) |
| 45 | def test_shuffle_rows_permutation( |
| 46 | num_tokens: int, hidden_size: int, dtype: torch.dtype |
| 47 | ): |
| 48 | """Test shuffle_rows with actual permutation.""" |
| 49 | if not current_platform.is_cuda(): |
| 50 | pytest.skip("shuffle_rows requires CUDA") |
| 51 | |
| 52 | # Create input tensor |
| 53 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 54 | |
| 55 | # Create a reverse permutation map |
| 56 | dst2src_map = torch.arange(num_tokens - 1, -1, -1, device="cuda", dtype=torch.int32) |
| 57 | |
| 58 | # Test shuffle_rows |
| 59 | output = shuffle_rows(input_tensor, dst2src_map) |
| 60 | |
| 61 | # Check that the output is the reverse of the input |
| 62 | expected_output = torch.flip(input_tensor, dims=[0]) |
| 63 | torch.testing.assert_close(output, expected_output, atol=1e-6, rtol=1e-5) |
| 64 | |
| 65 | # Check output shape and properties |
| 66 | assert output.shape == (num_tokens, hidden_size) |
| 67 | assert output.dtype == dtype |
| 68 | assert output.device == input_tensor.device |
| 69 | |
| 70 | |
| 71 | @pytest.mark.parametrize("num_tokens", [32, 64]) |
| 72 | @pytest.mark.parametrize("hidden_size", [256, 512]) |
| 73 | def test_shuffle_rows_expansion(num_tokens: int, hidden_size: int): |
| 74 | """Test shuffle_rows with expansion (more output tokens than input |
| 75 | tokens).""" |
| 76 | if not current_platform.is_cuda(): |
| 77 | pytest.skip("shuffle_rows requires CUDA") |
| 78 | |
| 79 | dtype = torch.float16 |
| 80 | |
| 81 | # Create input tensor |
| 82 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 83 | |
| 84 | # Create a mapping that duplicates some tokens (expansion) |
| 85 | expanded_size = num_tokens * 2 |
| 86 | dst2src_map = torch.randint( |
| 87 | 0, num_tokens, (expanded_size,), device="cuda", dtype=torch.int32 |
| 88 | ) |
| 89 | |
| 90 | # Test shuffle_rows |
| 91 | output = shuffle_rows(input_tensor, dst2src_map) |
| 92 | |
| 93 | # Check output shape |
| 94 | assert output.shape == (expanded_size, hidden_size) |
| 95 | assert output.dtype == dtype |
| 96 | assert output.device == input_tensor.device |
| 97 | |
| 98 | # Verify that each output row matches the corresponding input row |
| 99 | for i in range(expanded_size): |
| 100 | src_idx = dst2src_map[i].item() |
| 101 | torch.testing.assert_close( |
| 102 | output[i], input_tensor[src_idx], atol=1e-6, rtol=1e-5 |
| 103 | ) |
| 104 | |
| 105 | |
| 106 | def test_shuffle_rows_invalid_indices_are_zeroed(): |
| 107 | """Invalid sentinel indices should produce zero rows instead of OOB reads.""" |
| 108 | if not current_platform.is_cuda(): |
| 109 | pytest.skip("shuffle_rows requires CUDA") |
| 110 | |
| 111 | input_tensor = torch.randn(3, 128, device="cuda", dtype=torch.bfloat16) |
| 112 | dst2src_map = torch.tensor([-1, 1, 3], device="cuda", dtype=torch.int32) |
| 113 | |
| 114 | output = shuffle_rows(input_tensor, dst2src_map) |
| 115 | |
| 116 | torch.testing.assert_close(output[0], torch.zeros_like(output[0])) |
| 117 | torch.testing.assert_close(output[1], input_tensor[1]) |
| 118 | torch.testing.assert_close(output[2], torch.zeros_like(output[2])) |
| 119 | |
| 120 | |
| 121 | @pytest.mark.parametrize("num_tokens", [16, 64]) |
| 122 | @pytest.mark.parametrize("hidden_size", [128, 512]) |
| 123 | def test_shuffle_rows_random_permutation(num_tokens: int, hidden_size: int): |
| 124 | """Test shuffle_rows with random permutation.""" |
| 125 | if not current_platform.is_cuda(): |
| 126 | pytest.skip("shuffle_rows requires CUDA") |
| 127 | |
| 128 | dtype = torch.float16 |
| 129 | |
| 130 | # Set seed for reproducibility |
| 131 | torch.manual_seed(42) |
| 132 | |
| 133 | # Create input tensor |
| 134 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 135 | |
| 136 | # Create a random permutation map |
| 137 | dst2src_map = torch.randperm(num_tokens, device="cuda", dtype=torch.int32) |
| 138 | |
| 139 | # Test shuffle_rows |
| 140 | output = shuffle_rows(input_tensor, dst2src_map) |
| 141 | |
| 142 | # Check output shape and properties |
| 143 | assert output.shape == (num_tokens, hidden_size) |
| 144 | assert output.dtype == dtype |
| 145 | assert output.device == input_tensor.device |
| 146 | |
| 147 | # Verify that each output row matches the corresponding input row |
| 148 | for i in range(num_tokens): |
| 149 | src_idx = dst2src_map[i].item() |
| 150 | torch.testing.assert_close( |
| 151 | output[i], input_tensor[src_idx], atol=1e-6, rtol=1e-5 |
| 152 | ) |
| 153 | |
| 154 | |
| 155 | def test_shuffle_rows_edge_cases(): |
| 156 | """Test shuffle_rows with edge cases.""" |
| 157 | if not current_platform.is_cuda(): |
| 158 | pytest.skip("shuffle_rows requires CUDA") |
| 159 | |
| 160 | dtype = torch.float16 |
| 161 | |
| 162 | # Test with single token |
| 163 | input_tensor = torch.randn(1, 128, device="cuda", dtype=dtype) |
| 164 | dst2src_map = torch.tensor([0], device="cuda", dtype=torch.int32) |
| 165 | output = shuffle_rows(input_tensor, dst2src_map) |
| 166 | torch.testing.assert_close(output, input_tensor, atol=0, rtol=0) |
| 167 | |
| 168 | # Test with single feature dimension |
| 169 | input_tensor = torch.randn(16, 1, device="cuda", dtype=dtype) |
| 170 | dst2src_map = torch.arange(16, device="cuda", dtype=torch.int32) |
| 171 | output = shuffle_rows(input_tensor, dst2src_map) |
| 172 | torch.testing.assert_close(output, input_tensor, atol=0, rtol=0) |
| 173 | |
| 174 | |
| 175 | def test_shuffle_rows_moe_like_scenario(): |
| 176 | """Test shuffle_rows in a scenario similar to MoE usage.""" |
| 177 | if not current_platform.is_cuda(): |
| 178 | pytest.skip("shuffle_rows requires CUDA") |
| 179 | |
| 180 | dtype = torch.float16 |
| 181 | batch_size = 32 |
| 182 | hidden_size = 1024 |
| 183 | topk = 2 |
| 184 | |
| 185 | # Simulate input tokens |
| 186 | input_tensor = torch.randn(batch_size, hidden_size, device="cuda", dtype=dtype) |
| 187 | |
| 188 | # Simulate expert assignment (each token goes to topk experts) |
| 189 | # This creates a mapping where tokens are duplicated for multiple experts |
| 190 | total_tokens = batch_size * topk |
| 191 | dst2src_map = torch.zeros(total_tokens, device="cuda", dtype=torch.int32) |
| 192 | |
| 193 | # Fill the mapping to simulate MoE token distribution |
| 194 | for i in range(batch_size): |
| 195 | for k in range(topk): |
| 196 | dst2src_map[i * topk + k] = i |
| 197 | |
| 198 | # Test shuffle_rows |
| 199 | output = shuffle_rows(input_tensor, dst2src_map) |
| 200 | |
| 201 | # Check output shape |
| 202 | assert output.shape == (total_tokens, hidden_size) |
| 203 | assert output.dtype == dtype |
| 204 | assert output.device == input_tensor.device |
| 205 | |
| 206 | # Verify that tokens are correctly duplicated |
| 207 | for i in range(batch_size): |
| 208 | for k in range(topk): |
| 209 | output_idx = i * topk + k |
| 210 | torch.testing.assert_close( |
| 211 | output[output_idx], input_tensor[i], atol=1e-6, rtol=1e-5 |
| 212 | ) |
| 213 | |
| 214 | |
| 215 | @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) |
| 216 | def test_shuffle_rows_dtype_consistency(dtype: torch.dtype): |
| 217 | """Test that shuffle_rows preserves dtype correctly.""" |
| 218 | if not current_platform.is_cuda(): |
| 219 | pytest.skip("shuffle_rows requires CUDA") |
| 220 | |
| 221 | num_tokens = 64 |
| 222 | hidden_size = 512 |
| 223 | |
| 224 | # Create input tensor with specific dtype |
| 225 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 226 | dst2src_map = torch.arange(num_tokens, device="cuda", dtype=torch.int32) |
| 227 | |
| 228 | # Test shuffle_rows |
| 229 | output = shuffle_rows(input_tensor, dst2src_map) |
| 230 | |
| 231 | # Verify dtype is preserved |
| 232 | assert output.dtype == dtype |
| 233 | assert output.device == input_tensor.device |
| 234 | torch.testing.assert_close(output, input_tensor, atol=1e-6, rtol=1e-5) |
| 235 | |
| 236 | |
| 237 | def test_shuffle_rows_device_consistency(): |
| 238 | """Test that shuffle_rows maintains device consistency.""" |
| 239 | if not current_platform.is_cuda(): |
| 240 | pytest.skip("shuffle_rows requires CUDA") |
| 241 | |
| 242 | num_tokens = 32 |
| 243 | hidden_size = 256 |
| 244 | dtype = torch.float16 |
| 245 | |
| 246 | # Create input tensor on CUDA |
| 247 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 248 | dst2src_map = torch.arange(num_tokens, device="cuda", dtype=torch.int32) |
| 249 | |
| 250 | # Test shuffle_rows |
| 251 | output = shuffle_rows(input_tensor, dst2src_map) |
| 252 | |
| 253 | # Verify device is maintained |
| 254 | assert output.device == input_tensor.device |
| 255 | assert output.device.type == "cuda" |
| 256 | |
| 257 | |
| 258 | def test_shuffle_rows_contiguous_output(): |
| 259 | """Test that shuffle_rows produces contiguous output.""" |
| 260 | if not current_platform.is_cuda(): |
| 261 | pytest.skip("shuffle_rows requires CUDA") |
| 262 | |
| 263 | num_tokens = 64 |
| 264 | hidden_size = 512 |
| 265 | dtype = torch.float16 |
| 266 | |
| 267 | # Create input tensor |
| 268 | input_tensor = torch.randn(num_tokens, hidden_size, device="cuda", dtype=dtype) |
| 269 | dst2src_map = torch.arange(num_tokens, device="cuda", dtype=torch.int32) |
| 270 | |
| 271 | # Test shuffle_rows |
| 272 | output = shuffle_rows(input_tensor, dst2src_map) |
| 273 | |
| 274 | # Verify output is contiguous |
| 275 | assert output.is_contiguous() |