| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import math |
| 5 | |
| 6 | import pytest |
| 7 | import torch |
| 8 | import torch.nn.functional as F |
| 9 | |
| 10 | from vllm.platforms import current_platform |
| 11 | |
| 12 | if not ( |
| 13 | current_platform.is_cuda() and current_platform.is_device_capability_family(100) |
| 14 | ): |
| 15 | pytest.skip( |
| 16 | reason="GDN CuteDSL prefill requires CUDA SM10x.", |
| 17 | allow_module_level=True, |
| 18 | ) |
| 19 | |
| 20 | from vllm.model_executor.layers.mamba.ops.gdn_chunk_cutedsl import ( # noqa: E402 |
| 21 | chunk_gated_delta_rule_cutedsl, |
| 22 | prepare_metadata_cutedsl, |
| 23 | ) |
| 24 | from vllm.third_party.flash_linear_attention.ops import ( # noqa: E402 |
| 25 | chunk_gated_delta_rule, |
| 26 | ) |
| 27 | from vllm.third_party.flash_linear_attention.ops.index import ( # noqa: E402 |
| 28 | prepare_chunk_indices, |
| 29 | prepare_chunk_offsets, |
| 30 | ) |
| 31 | |
| 32 | |
| 33 | @pytest.mark.parametrize("num_seqs", [1, 5, 257]) |
| 34 | @pytest.mark.parametrize("state_dtype", [torch.bfloat16, torch.float32]) |
| 35 | def test_gdn_chunk_cutedsl_correctness(num_seqs: int, state_dtype: torch.dtype): |
| 36 | rng_cpu = torch.Generator("cpu").manual_seed(1234) |
| 37 | rng = torch.Generator("cuda").manual_seed(2345) |
| 38 | |
| 39 | seq_lens = torch.randint(1, 130, (num_seqs,), dtype=torch.int32, generator=rng_cpu) |
| 40 | cu_seqlens = torch.zeros(num_seqs + 1, device="cuda", dtype=torch.int32) |
| 41 | cu_seqlens[1:] = seq_lens.to(device="cuda").cumsum(0) |
| 42 | total_tokens = int(cu_seqlens[-1].item()) |
| 43 | |
| 44 | num_k_heads = 4 |
| 45 | num_v_heads = 8 |
| 46 | head_k_dim = 128 |
| 47 | head_v_dim = 128 |
| 48 | dtype = torch.bfloat16 |
| 49 | |
| 50 | q = torch.randn( |
| 51 | 1, |
| 52 | total_tokens, |
| 53 | num_k_heads, |
| 54 | head_k_dim, |
| 55 | device="cuda", |
| 56 | dtype=dtype, |
| 57 | generator=rng, |
| 58 | ) |
| 59 | k = torch.randn_like(q, generator=rng) |
| 60 | v = torch.randn( |
| 61 | 1, |
| 62 | total_tokens, |
| 63 | num_v_heads, |
| 64 | head_v_dim, |
| 65 | device="cuda", |
| 66 | dtype=dtype, |
| 67 | generator=rng, |
| 68 | ) |
| 69 | q = F.normalize(q.float(), p=2, dim=-1).to(dtype) |
| 70 | k = F.normalize(k.float(), p=2, dim=-1).to(dtype) |
| 71 | a = torch.randn( |
| 72 | 1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng |
| 73 | ) |
| 74 | b = torch.randn( |
| 75 | 1, total_tokens, num_v_heads, device="cuda", dtype=dtype, generator=rng |
| 76 | ) |
| 77 | # Match upstream FLA GatedDeltaNet synthetic initialization: |
| 78 | # https://github.com/fla-org/flash-linear-attention/blob/main/fla/layers/gated_deltanet.py |
| 79 | A = torch.empty(num_v_heads, device="cuda", dtype=torch.float32).uniform_( |
| 80 | 0, 16, generator=rng |
| 81 | ) |
| 82 | A_log = torch.log(A) |
| 83 | dt = torch.exp( |
| 84 | torch.rand(num_v_heads, device="cuda", dtype=torch.float32, generator=rng) |
| 85 | * (math.log(0.1) - math.log(0.001)) |
| 86 | + math.log(0.001) |
| 87 | ) |
| 88 | dt = torch.clamp(dt, min=1e-4) |
| 89 | dt_bias = dt + torch.log(-torch.expm1(-dt)) |
| 90 | g = -A_log.exp().view(1, 1, num_v_heads) * F.softplus( |
| 91 | a.float() + dt_bias.view(1, 1, num_v_heads) |
| 92 | ) |
| 93 | beta = torch.sigmoid(b.float()) |
| 94 | initial_state = ( |
| 95 | torch.randn( |
| 96 | num_seqs, |
| 97 | num_v_heads, |
| 98 | head_v_dim, |
| 99 | head_k_dim, |
| 100 | device="cuda", |
| 101 | dtype=state_dtype, |
| 102 | generator=rng, |
| 103 | ) |
| 104 | * 0.05 |
| 105 | ) |
| 106 | |
| 107 | # check metadata kernel |
| 108 | chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens) |
| 109 | torch.accelerator.synchronize() |
| 110 | |
| 111 | expected_indices = prepare_chunk_indices(cu_seqlens, 64) |
| 112 | expected_offsets = prepare_chunk_offsets(cu_seqlens, 64) |
| 113 | total_chunks = int(expected_offsets[-1].item()) |
| 114 | |
| 115 | torch.testing.assert_close(chunk_offsets, expected_offsets.to(torch.int32)) |
| 116 | torch.testing.assert_close( |
| 117 | chunk_indices[:total_chunks], |
| 118 | expected_indices, |
| 119 | ) |
| 120 | |
| 121 | ref_o, ref_state = chunk_gated_delta_rule( |
| 122 | q=q, |
| 123 | k=k, |
| 124 | v=v, |
| 125 | g=g, |
| 126 | beta=beta, |
| 127 | initial_state=initial_state, |
| 128 | output_final_state=True, |
| 129 | cu_seqlens=cu_seqlens, |
| 130 | use_qk_l2norm_in_kernel=False, |
| 131 | ) |
| 132 | actual_core_attn_out = torch.empty( |
| 133 | total_tokens, |
| 134 | num_v_heads, |
| 135 | head_v_dim, |
| 136 | device="cuda", |
| 137 | dtype=dtype, |
| 138 | ) |
| 139 | actual_o, actual_state = chunk_gated_delta_rule_cutedsl( |
| 140 | q=q, |
| 141 | k=k, |
| 142 | v=v, |
| 143 | g=g, |
| 144 | beta=beta, |
| 145 | initial_state=initial_state, |
| 146 | cu_seqlens=cu_seqlens, |
| 147 | chunk_indices=chunk_indices, |
| 148 | chunk_offsets=chunk_offsets, |
| 149 | core_attn_out=actual_core_attn_out, |
| 150 | ) |
| 151 | torch.accelerator.synchronize() |
| 152 | |
| 153 | # check main kernel |
| 154 | o_error = (actual_o.float() - ref_o.float()).abs() |
| 155 | state_error = ( |
| 156 | actual_state.float() - ref_state.to(actual_state.dtype).float() |
| 157 | ).abs() |
| 158 | assert o_error.max().item() < 2e-3 |
| 159 | assert o_error.mean().item() < 6e-5 |
| 160 | assert state_error.max().item() < 2e-2 |
| 161 | assert state_error.mean().item() < 6e-4 |
| 162 | core_attn_out_error = ( |
| 163 | actual_core_attn_out.float() - actual_o.squeeze(0).float() |
| 164 | ).abs() |
| 165 | assert core_attn_out_error.max().item() == 0 |
| 166 | |
| 167 | # check main kernel when core_attn_out is not passed |
| 168 | no_buffer_o, no_buffer_state = chunk_gated_delta_rule_cutedsl( |
| 169 | q=q, |
| 170 | k=k, |
| 171 | v=v, |
| 172 | g=g, |
| 173 | beta=beta, |
| 174 | initial_state=initial_state, |
| 175 | cu_seqlens=cu_seqlens, |
| 176 | chunk_indices=chunk_indices, |
| 177 | chunk_offsets=chunk_offsets, |
| 178 | ) |
| 179 | torch.accelerator.synchronize() |
| 180 | |
| 181 | no_buffer_o_error = (no_buffer_o.float() - ref_o.float()).abs() |
| 182 | no_buffer_state_error = ( |
| 183 | no_buffer_state.float() - ref_state.to(no_buffer_state.dtype).float() |
| 184 | ).abs() |
| 185 | buffer_o_error = (no_buffer_o.float() - actual_o.float()).abs() |
| 186 | buffer_state_error = ( |
| 187 | no_buffer_state.float() - actual_state.to(no_buffer_state.dtype).float() |
| 188 | ).abs() |
| 189 | assert no_buffer_o_error.max().item() < 2e-3 |
| 190 | assert no_buffer_o_error.mean().item() < 6e-5 |
| 191 | assert no_buffer_state_error.max().item() < 2e-2 |
| 192 | assert no_buffer_state_error.mean().item() < 6e-4 |
| 193 | assert buffer_o_error.max().item() == 0 |
| 194 | assert buffer_state_error.max().item() == 0 |
| 195 | |
| 196 | |
| 197 | def test_gdn_chunk_cutedsl_correlated_keys_stability(): |
| 198 | """Cover a KKT block where three BF16 Newton rounds diverge.""" |
| 199 | total_tokens = 64 |
| 200 | num_k_heads = 4 |
| 201 | num_v_heads = 8 |
| 202 | head_dim = 128 |
| 203 | rng = torch.Generator("cuda").manual_seed(3456) |
| 204 | |
| 205 | q = torch.zeros( |
| 206 | 1, |
| 207 | total_tokens, |
| 208 | num_k_heads, |
| 209 | head_dim, |
| 210 | device="cuda", |
| 211 | dtype=torch.bfloat16, |
| 212 | ) |
| 213 | k = torch.zeros_like(q) |
| 214 | q[..., 0] = 1 |
| 215 | k[..., 0] = 1 |
| 216 | v = torch.randn( |
| 217 | 1, |
| 218 | total_tokens, |
| 219 | num_v_heads, |
| 220 | head_dim, |
| 221 | device="cuda", |
| 222 | dtype=torch.bfloat16, |
| 223 | generator=rng, |
| 224 | ) |
| 225 | g = torch.zeros( |
| 226 | 1, |
| 227 | total_tokens, |
| 228 | num_v_heads, |
| 229 | device="cuda", |
| 230 | dtype=torch.float32, |
| 231 | ) |
| 232 | beta = torch.full_like(g, 0.9) |
| 233 | initial_state = torch.zeros( |
| 234 | 1, |
| 235 | num_v_heads, |
| 236 | head_dim, |
| 237 | head_dim, |
| 238 | device="cuda", |
| 239 | dtype=torch.float32, |
| 240 | ) |
| 241 | cu_seqlens = torch.tensor([0, total_tokens], device="cuda", dtype=torch.int32) |
| 242 | chunk_indices, chunk_offsets = prepare_metadata_cutedsl(cu_seqlens, total_tokens) |
| 243 | |
| 244 | ref_o, ref_state = chunk_gated_delta_rule( |
| 245 | q=q, |
| 246 | k=k, |
| 247 | v=v, |
| 248 | g=g, |
| 249 | beta=beta, |
| 250 | initial_state=initial_state, |
| 251 | output_final_state=True, |
| 252 | cu_seqlens=cu_seqlens, |
| 253 | use_qk_l2norm_in_kernel=False, |
| 254 | ) |
| 255 | actual_o, actual_state = chunk_gated_delta_rule_cutedsl( |
| 256 | q=q, |
| 257 | k=k, |
| 258 | v=v, |
| 259 | g=g, |
| 260 | beta=beta, |
| 261 | initial_state=initial_state, |
| 262 | cu_seqlens=cu_seqlens, |
| 263 | chunk_indices=chunk_indices, |
| 264 | chunk_offsets=chunk_offsets, |
| 265 | ) |
| 266 | torch.accelerator.synchronize() |
| 267 | |
| 268 | assert torch.isfinite(actual_o).all() |
| 269 | assert torch.isfinite(actual_state).all() |
| 270 | output_error = (actual_o.float() - ref_o.float()).abs() |
| 271 | state_error = ( |
| 272 | actual_state.float() - ref_state.to(actual_state.dtype).float() |
| 273 | ).abs() |
| 274 | assert output_error.max().item() < 1e-1 |
| 275 | assert output_error.mean().item() < 2e-3 |
| 276 | assert state_error.max().item() < 2e-1 |
| 277 | assert state_error.mean().item() < 6e-4 |