| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | import torch |
| 5 | |
| 6 | |
| 7 | @pytest.mark.parametrize("sm120", [False, True]) |
| 8 | def test_deepseek_v4_c128a_adaptive_width_has_capture_stable_stride( |
| 9 | monkeypatch: pytest.MonkeyPatch, |
| 10 | sm120: bool, |
| 11 | ): |
| 12 | from vllm.models.deepseek_v4 import sparse_mla |
| 13 | from vllm.platforms.interface import DeviceCapability |
| 14 | |
| 15 | monkeypatch.setattr( |
| 16 | sparse_mla.current_platform, |
| 17 | "get_device_capability", |
| 18 | lambda: DeviceCapability(12, 0) if sm120 else DeviceCapability(10, 0), |
| 19 | ) |
| 20 | |
| 21 | device = torch.device("cuda") |
| 22 | capacity_width = 512 |
| 23 | global_decode_buffer = torch.empty( |
| 24 | (2, capacity_width), dtype=torch.int32, device=device |
| 25 | ) |
| 26 | prefill_buffer = torch.empty_like(global_decode_buffer) |
| 27 | kwargs = dict( |
| 28 | positions=torch.tensor([255, 511, 383, 639], device=device), |
| 29 | compress_ratio=128, |
| 30 | num_decode_tokens=2, |
| 31 | token_to_req_indices=torch.tensor( |
| 32 | [0, 1, 0, 1], dtype=torch.int32, device=device |
| 33 | ), |
| 34 | block_table=torch.tensor([[3], [5]], dtype=torch.int32, device=device), |
| 35 | block_size=capacity_width, |
| 36 | slot_mapping=torch.arange(4, dtype=torch.int64, device=device), |
| 37 | global_decode_buffer=global_decode_buffer, |
| 38 | decode_lens_buffer=torch.empty(2, dtype=torch.int32, device=device), |
| 39 | prefill_buffer=prefill_buffer, |
| 40 | ) |
| 41 | captured_decode, _, captured_prefill = sparse_mla.build_c128a_topk_metadata( |
| 42 | max_compressed_tokens=256, |
| 43 | **kwargs, |
| 44 | ) |
| 45 | # SM120 keeps the decode view contiguous across the full buffer width; |
| 46 | # other backends get the active-width slice. The prefill view is always |
| 47 | # narrowed. |
| 48 | expected_decode_width = capacity_width if sm120 else 256 |
| 49 | assert captured_decode.shape == (2, expected_decode_width) |
| 50 | assert captured_prefill.shape == (2, 256) |
| 51 | assert captured_decode.stride(0) == captured_prefill.stride(0) == capacity_width |
| 52 | assert captured_decode.is_contiguous() == sm120 |
| 53 | |
| 54 | captured_rows = torch.empty((4, 4), dtype=torch.int32, device=device) |
| 55 | captured_rows[:2].copy_(captured_decode[:, :4]) |
| 56 | captured_rows[2:].copy_(captured_prefill[:, :4]) |
| 57 | torch.accelerator.synchronize() |
| 58 | graph = torch.cuda.CUDAGraph() |
| 59 | with torch.cuda.graph(graph): |
| 60 | captured_rows[:2].copy_(captured_decode[:, :4]) |
| 61 | captured_rows[2:].copy_(captured_prefill[:, :4]) |
| 62 | |
| 63 | global_decode_buffer.fill_(-99) |
| 64 | prefill_buffer.fill_(-99) |
| 65 | sparse_mla.build_c128a_topk_metadata( |
| 66 | max_compressed_tokens=128, |
| 67 | **kwargs, |
| 68 | ) |
| 69 | graph.replay() |
| 70 | |
| 71 | assert captured_rows.cpu().tolist() == [ |
| 72 | [1536, 1537, -1, -1], |
| 73 | [2560, 2561, 2562, 2563], |
| 74 | [0, 1, 2, -1], |
| 75 | [0, 1, 2, 3], |
| 76 | ] |
| 77 | assert torch.all(global_decode_buffer[:, 128:] == -99) |
| 78 | assert torch.all(prefill_buffer[:, 128:] == -99) |
| 79 | |
| 80 | |
| 81 | def test_sparse_flashmla_metadata_smoke(): |
| 82 | import vllm.v1.attention.ops.flashmla as fm |
| 83 | |
| 84 | ok, reason = fm.is_flashmla_sparse_supported() |
| 85 | if not ok: |
| 86 | pytest.skip(reason) |
| 87 | |
| 88 | device = torch.device("cuda") |
| 89 | batch_size = 1 |
| 90 | seqlen_q = 1 |
| 91 | num_heads_q = 128 |
| 92 | num_heads_k = 1 |
| 93 | q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k |
| 94 | topk = 128 |
| 95 | |
| 96 | cache_seqlens = torch.zeros(batch_size, dtype=torch.int32, device=device) |
| 97 | |
| 98 | tile_md, num_splits = fm.get_mla_metadata( |
| 99 | cache_seqlens, |
| 100 | q_seq_per_hk, |
| 101 | num_heads_k, |
| 102 | num_heads_q=num_heads_q, |
| 103 | topk=topk, |
| 104 | is_fp8_kvcache=True, |
| 105 | ) |
| 106 | assert isinstance(tile_md, fm.FlashMLASchedMeta) |
| 107 | assert tile_md.tile_scheduler_metadata is None |
| 108 | assert tile_md.num_splits is None |
| 109 | assert num_splits is None |
| 110 | |
| 111 | |
| 112 | def test_sparse_flashmla_decode_matches_cache_writer_scales(): |
| 113 | import vllm.v1.attention.ops.flashmla as fm |
| 114 | from vllm import _custom_ops as ops |
| 115 | |
| 116 | ok, reason = fm.is_flashmla_sparse_supported() |
| 117 | if not ok: |
| 118 | pytest.skip(reason) |
| 119 | |
| 120 | device = torch.device("cuda") |
| 121 | batch_size = 1 |
| 122 | seqlen_q = 1 |
| 123 | num_heads_q = 64 |
| 124 | head_dim_k = 576 |
| 125 | head_dim_v = 512 |
| 126 | num_heads_k = 1 |
| 127 | page_block_size = 64 |
| 128 | bytes_per_token = 656 |
| 129 | topk = 128 |
| 130 | |
| 131 | # Metadata |
| 132 | q_seq_per_hk = seqlen_q * num_heads_q // num_heads_k |
| 133 | # q_heads_per_hk = num_heads_q // num_heads_k |
| 134 | cache_seqlens = torch.ones(batch_size, dtype=torch.int32, device=device) |
| 135 | tile_md, num_splits = fm.get_mla_metadata( |
| 136 | cache_seqlens, |
| 137 | q_seq_per_hk, |
| 138 | num_heads_k, |
| 139 | num_heads_q=num_heads_q, |
| 140 | topk=topk, |
| 141 | is_fp8_kvcache=True, |
| 142 | ) |
| 143 | |
| 144 | # Inputs |
| 145 | q = torch.zeros( |
| 146 | (batch_size, seqlen_q, num_heads_q, head_dim_k), |
| 147 | dtype=torch.bfloat16, |
| 148 | device=device, |
| 149 | ) |
| 150 | cache_rows = torch.zeros( |
| 151 | (1, page_block_size, bytes_per_token), dtype=torch.uint8, device=device |
| 152 | ) |
| 153 | # 336 / 448 = 0.75 distinguishes arbitrary FP32 from power-of-two scales. |
| 154 | kv_c = torch.full((1, 512), 336.0, dtype=torch.bfloat16, device=device) |
| 155 | kv_c[:, -128:] = 0 |
| 156 | ops.concat_and_cache_mla( |
| 157 | kv_c, |
| 158 | torch.zeros((1, 64), dtype=torch.bfloat16, device=device), |
| 159 | cache_rows, |
| 160 | torch.zeros(1, dtype=torch.int64, device=device), |
| 161 | "fp8_ds_mla", |
| 162 | torch.ones(1, dtype=torch.float32, device=device), |
| 163 | ) |
| 164 | scales = cache_rows[0, 0].view(torch.float32)[128:132] |
| 165 | expected_scales = torch.tensor([1.0, 1.0, 1.0, 2**-13], device=device) |
| 166 | torch.testing.assert_close(scales, expected_scales, rtol=0, atol=0) |
| 167 | cached_nope = cache_rows[0, 0, :512].view(torch.float8_e4m3fn).float() |
| 168 | expected = (cached_nope * scales.repeat_interleave(128)).to(torch.bfloat16) |
| 169 | |
| 170 | k_cache = cache_rows.unsqueeze(2) |
| 171 | indices = torch.full( |
| 172 | (batch_size, seqlen_q, topk), -1, dtype=torch.int32, device=device |
| 173 | ) |
| 174 | indices[..., 0] = 0 |
| 175 | # SM90 sparse decode only supports a fixed top-k width. Padding with -1 |
| 176 | # exercises the same cache data on SM90 and SM100 without topk_length. |
| 177 | |
| 178 | block_table = torch.zeros((batch_size, 128), dtype=torch.int32, device=device) |
| 179 | out, lse = fm.flash_mla_with_kvcache( |
| 180 | q, |
| 181 | k_cache, |
| 182 | block_table, |
| 183 | cache_seqlens, |
| 184 | head_dim_v, |
| 185 | tile_md, |
| 186 | num_splits, |
| 187 | indices=indices, |
| 188 | is_fp8_kvcache=True, |
| 189 | ) |
| 190 | assert out.shape[0] == batch_size |
| 191 | assert out.shape[-1] == head_dim_v |
| 192 | assert lse.shape[0] == batch_size |
| 193 | torch.testing.assert_close( |
| 194 | out, expected.view(1, 1, 1, 512).expand_as(out), rtol=0, atol=0 |
| 195 | ) |
| 196 | |
| 197 | |
| 198 | @pytest.mark.parametrize("h_q", [64, 128]) |
| 199 | def test_sparse_flashmla_prefill_smoke(h_q: int): |
| 200 | import vllm.v1.attention.ops.flashmla as fm |
| 201 | |
| 202 | ok, reason = fm.is_flashmla_sparse_supported() |
| 203 | if not ok: |
| 204 | pytest.skip(reason) |
| 205 | |
| 206 | device = torch.device("cuda") |
| 207 | torch.manual_seed(0) |
| 208 | s_q = 1 |
| 209 | s_kv = 8 |
| 210 | h_kv = 1 |
| 211 | d_qk = 576 |
| 212 | d_v = 512 |
| 213 | topk = 128 |
| 214 | q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device) |
| 215 | kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device) |
| 216 | indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device) |
| 217 | reference_indices = indices.clone() |
| 218 | reference_indices[..., 1:] = -1 |
| 219 | kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)} |
| 220 | reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs) |
| 221 | actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs) |
| 222 | |
| 223 | for actual_tensor, reference_tensor in zip(actual, reference): |
| 224 | torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0) |
| 225 | assert actual[0].shape == (s_q, h_q, d_v) |
| 226 | |
| 227 | |
| 228 | def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences(): |
| 229 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 230 | |
| 231 | metadata = DeepseekSparseSWAMetadata( |
| 232 | block_table=torch.empty(0, dtype=torch.int32), |
| 233 | slot_mapping=torch.empty(0, dtype=torch.int32), |
| 234 | block_size=64, |
| 235 | num_prefills=5, |
| 236 | prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32), |
| 237 | prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32), |
| 238 | prefill_window_size=64, |
| 239 | prefill_max_model_len=1024, |
| 240 | prefill_max_num_batched_tokens=128, |
| 241 | ) |
| 242 | |
| 243 | chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4) |
| 244 | |
| 245 | # the adaptive plan keeps all 5 in one chunk |
| 246 | assert chunk_plan == [(0, 5, 36, 103)] |
| 247 | |
| 248 | |
| 249 | def test_flashinfer_sparse_indices_cache(monkeypatch): |
| 250 | from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod |
| 251 | from vllm.models.deepseek_v4.sparse_mla import DeepseekV4FlashMLAMetadata |
| 252 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 253 | |
| 254 | builder_calls = 0 |
| 255 | |
| 256 | def fake_build(*args, **kwargs): |
| 257 | nonlocal builder_calls |
| 258 | builder_calls += 1 |
| 259 | return ( |
| 260 | torch.tensor([[builder_calls]], dtype=torch.int32), |
| 261 | torch.tensor([builder_calls], dtype=torch.int32), |
| 262 | ) |
| 263 | |
| 264 | monkeypatch.setattr( |
| 265 | flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build |
| 266 | ) |
| 267 | |
| 268 | def make_attn(compress_ratio: int, topk_width: int): |
| 269 | attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention) |
| 270 | attn.compress_ratio = compress_ratio |
| 271 | attn.window_size = 4 |
| 272 | attn.topk_indices_buffer = torch.tensor( |
| 273 | [[0, 1], [2, 3], [4, 5]], dtype=torch.int32 |
| 274 | )[:, :topk_width] |
| 275 | return attn |
| 276 | |
| 277 | def make_swa_metadata(): |
| 278 | return DeepseekSparseSWAMetadata( |
| 279 | block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32), |
| 280 | slot_mapping=torch.tensor([0, 1], dtype=torch.int64), |
| 281 | block_size=64, |
| 282 | seq_lens=torch.tensor([8, 10], dtype=torch.int32), |
| 283 | query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32), |
| 284 | query_start_loc_cpu=torch.tensor([0, 1, 3], dtype=torch.int32), |
| 285 | token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32), |
| 286 | decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32), |
| 287 | decode_swa_lens=torch.tensor([2], dtype=torch.int32), |
| 288 | decode_swa_width=4, |
| 289 | is_valid_token=torch.tensor([True], dtype=torch.bool), |
| 290 | num_decodes=1, |
| 291 | num_prefills=1, |
| 292 | num_decode_tokens=1, |
| 293 | num_prefill_tokens=2, |
| 294 | ) |
| 295 | |
| 296 | def make_flashmla_metadata(): |
| 297 | return DeepseekV4FlashMLAMetadata( |
| 298 | num_reqs=2, |
| 299 | max_query_len=2, |
| 300 | max_seq_len=10, |
| 301 | num_actual_tokens=3, |
| 302 | query_start_loc=torch.tensor([0, 1, 3], dtype=torch.int32), |
| 303 | slot_mapping=torch.tensor([0, 1, 2], dtype=torch.int64), |
| 304 | block_table=torch.tensor([[0, 1], [2, 3]], dtype=torch.int32), |
| 305 | req_id_per_token=torch.tensor([0, 1, 1], dtype=torch.int32), |
| 306 | block_size=256, |
| 307 | topk_tokens=2, |
| 308 | c128a_global_decode_topk_indices=torch.tensor( |
| 309 | [[[9, 10]]], dtype=torch.int32 |
| 310 | ), |
| 311 | c128a_decode_topk_lens=torch.tensor([2], dtype=torch.int32), |
| 312 | c128a_prefill_topk_indices=torch.tensor( |
| 313 | [[0, 1], [1, 2]], dtype=torch.int32 |
| 314 | ), |
| 315 | ) |
| 316 | |
| 317 | swa_attn = make_attn(1, 0) |
| 318 | swa_metadata = make_swa_metadata() |
| 319 | _, _, sparse_indices_first, sparse_lens_first = ( |
| 320 | swa_attn._build_sparse_index_metadata( |
| 321 | kv_cache=None, |
| 322 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 323 | swa_metadata=swa_metadata, |
| 324 | attn_metadata=None, |
| 325 | swa_only=True, |
| 326 | ) |
| 327 | ) |
| 328 | _, _, sparse_indices_second, sparse_lens_second = ( |
| 329 | swa_attn._build_sparse_index_metadata( |
| 330 | kv_cache=None, |
| 331 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 332 | swa_metadata=swa_metadata, |
| 333 | attn_metadata=None, |
| 334 | swa_only=True, |
| 335 | ) |
| 336 | ) |
| 337 | assert builder_calls == 1 |
| 338 | assert sparse_indices_first is sparse_indices_second |
| 339 | assert sparse_lens_first is sparse_lens_second |
| 340 | |
| 341 | c128a_attn = make_attn(128, 2) |
| 342 | c128a_metadata = make_swa_metadata() |
| 343 | c128a_flashmla_md = make_flashmla_metadata() |
| 344 | _, _, sparse_indices_first, sparse_lens_first = ( |
| 345 | c128a_attn._build_sparse_index_metadata( |
| 346 | kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16), |
| 347 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 348 | swa_metadata=c128a_metadata, |
| 349 | attn_metadata=c128a_flashmla_md, |
| 350 | swa_only=False, |
| 351 | ) |
| 352 | ) |
| 353 | _, _, sparse_indices_second, sparse_lens_second = ( |
| 354 | c128a_attn._build_sparse_index_metadata( |
| 355 | kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16), |
| 356 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 357 | swa_metadata=c128a_metadata, |
| 358 | attn_metadata=c128a_flashmla_md, |
| 359 | swa_only=False, |
| 360 | ) |
| 361 | ) |
| 362 | |
| 363 | assert builder_calls == 2 |
| 364 | assert sparse_indices_first is sparse_indices_second |
| 365 | assert sparse_lens_first is sparse_lens_second |
| 366 | |
| 367 | c4a_attn = make_attn(4, 2) |
| 368 | c4a_metadata = make_swa_metadata() |
| 369 | c4a_flashmla_md = make_flashmla_metadata() |
| 370 | c4a_flashmla_md.c128a_global_decode_topk_indices = None |
| 371 | c4a_flashmla_md.c128a_decode_topk_lens = None |
| 372 | c4a_flashmla_md.c128a_prefill_topk_indices = None |
| 373 | _, _, sparse_indices_third, sparse_lens_third = ( |
| 374 | c4a_attn._build_sparse_index_metadata( |
| 375 | kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16), |
| 376 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 377 | swa_metadata=c4a_metadata, |
| 378 | attn_metadata=c4a_flashmla_md, |
| 379 | swa_only=False, |
| 380 | ) |
| 381 | ) |
| 382 | _, _, sparse_indices_fourth, sparse_lens_fourth = ( |
| 383 | c4a_attn._build_sparse_index_metadata( |
| 384 | kv_cache=torch.empty((1, 2, 512), dtype=torch.bfloat16), |
| 385 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 386 | swa_metadata=c4a_metadata, |
| 387 | attn_metadata=c4a_flashmla_md, |
| 388 | swa_only=False, |
| 389 | ) |
| 390 | ) |
| 391 | |
| 392 | assert builder_calls == 4 |
| 393 | assert sparse_indices_third is not sparse_indices_fourth |
| 394 | assert sparse_lens_third is not sparse_lens_fourth |
| 395 | |
| 396 | |
| 397 | def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch): |
| 398 | from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod |
| 399 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 400 | |
| 401 | captured_shapes_and_windows: list[tuple[int, int]] = [] |
| 402 | |
| 403 | def fake_build(*args, **kwargs): |
| 404 | # window_size is the 12th positional arg of |
| 405 | # build_flashinfer_mixed_sparse_indices. |
| 406 | captured_shapes_and_windows.append((args[0].shape[-1], args[11])) |
| 407 | num_tokens = args[0].shape[0] + args[3].shape[0] |
| 408 | return ( |
| 409 | torch.zeros((num_tokens, 1), dtype=torch.int32), |
| 410 | torch.zeros((num_tokens,), dtype=torch.int32), |
| 411 | ) |
| 412 | |
| 413 | monkeypatch.setattr( |
| 414 | flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build |
| 415 | ) |
| 416 | |
| 417 | attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention) |
| 418 | attn.compress_ratio = 1 |
| 419 | attn.window_size = 4 |
| 420 | attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32) |
| 421 | |
| 422 | wide_width = 8 |
| 423 | wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32) |
| 424 | wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32) |
| 425 | wide_metadata = DeepseekSparseSWAMetadata( |
| 426 | block_table=torch.tensor([[0, 1]], dtype=torch.int32), |
| 427 | slot_mapping=torch.tensor([0], dtype=torch.int64), |
| 428 | block_size=64, |
| 429 | seq_lens=torch.tensor([8], dtype=torch.int32), |
| 430 | query_start_loc=torch.tensor([0, 1], dtype=torch.int32), |
| 431 | query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32), |
| 432 | token_to_req_indices=torch.tensor([0], dtype=torch.int32), |
| 433 | decode_swa_indices=wide_indices, |
| 434 | decode_swa_lens=torch.tensor([2], dtype=torch.int32), |
| 435 | decode_swa_width=wide_width, |
| 436 | is_valid_token=torch.tensor([True], dtype=torch.bool), |
| 437 | num_decodes=1, |
| 438 | num_prefills=0, |
| 439 | num_decode_tokens=1, |
| 440 | num_prefill_tokens=0, |
| 441 | ) |
| 442 | attn._build_sparse_index_metadata( |
| 443 | kv_cache=None, |
| 444 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 445 | swa_metadata=wide_metadata, |
| 446 | attn_metadata=None, |
| 447 | swa_only=True, |
| 448 | ) |
| 449 | assert captured_shapes_and_windows == [(wide_width, attn.window_size)] |
| 450 | |
| 451 | empty_width = 8 |
| 452 | empty_metadata = DeepseekSparseSWAMetadata( |
| 453 | block_table=torch.tensor([[0, 1]], dtype=torch.int32), |
| 454 | slot_mapping=torch.tensor([0, 1], dtype=torch.int64), |
| 455 | block_size=64, |
| 456 | seq_lens=torch.tensor([8], dtype=torch.int32), |
| 457 | query_start_loc=torch.tensor([0, 2], dtype=torch.int32), |
| 458 | query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32), |
| 459 | token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32), |
| 460 | decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32), |
| 461 | decode_swa_lens=torch.empty((0,), dtype=torch.int32), |
| 462 | decode_swa_width=empty_width, |
| 463 | is_valid_token=torch.tensor([True, True], dtype=torch.bool), |
| 464 | num_decodes=0, |
| 465 | num_prefills=1, |
| 466 | num_decode_tokens=0, |
| 467 | num_prefill_tokens=2, |
| 468 | ) |
| 469 | attn._build_sparse_index_metadata( |
| 470 | kv_cache=None, |
| 471 | swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), |
| 472 | swa_metadata=empty_metadata, |
| 473 | attn_metadata=None, |
| 474 | swa_only=True, |
| 475 | ) |
| 476 | assert captured_shapes_and_windows == [ |
| 477 | (wide_width, attn.window_size), |
| 478 | (empty_width, attn.window_size), |
| 479 | ] |
| 480 | |
| 481 | |
| 482 | def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width(): |
| 483 | from vllm.models.deepseek_v4.common.ops.cache_utils import ( |
| 484 | build_flashinfer_mixed_sparse_indices, |
| 485 | ) |
| 486 | |
| 487 | device = torch.device("cuda") |
| 488 | padded_width = 8 |
| 489 | logical_window = 4 |
| 490 | sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( |
| 491 | decode_swa_indices=torch.empty( |
| 492 | (0, padded_width), dtype=torch.int32, device=device |
| 493 | ), |
| 494 | decode_compressed_indices=None, |
| 495 | decode_compressed_topk_lens=None, |
| 496 | prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device), |
| 497 | query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device), |
| 498 | seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device), |
| 499 | token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device), |
| 500 | swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), |
| 501 | swa_block_size=64, |
| 502 | compressed_block_table=None, |
| 503 | compressed_block_size=64, |
| 504 | window_size=logical_window, |
| 505 | compress_ratio=1, |
| 506 | topk=0, |
| 507 | ) |
| 508 | |
| 509 | assert sparse_indices.shape == (1, padded_width) |
| 510 | assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1] |
| 511 | assert sparse_lens.cpu().tolist() == [padded_width] |
| 512 | |
| 513 | |
| 514 | def test_flashinfer_mixed_sparse_indices_noncausal_rows_have_no_active_gaps(): |
| 515 | """A DSpark non-causal row keeps -1 out of the kernel's active ranges. |
| 516 | |
| 517 | Each draft token's causal window holds its first visible entries, the rest |
| 518 | of the block starts at column `window`, and the compressed entries follow |
| 519 | directly; `sparse_topk_lens` ends right after them. |
| 520 | """ |
| 521 | from vllm.models.deepseek_v4.common.ops.cache_utils import ( |
| 522 | build_flashinfer_mixed_sparse_indices, |
| 523 | ) |
| 524 | |
| 525 | device = torch.device("cuda") |
| 526 | window, width = 4, 8 |
| 527 | # One request: 1 context token, a 2-token draft block, slots 0..2 visible. |
| 528 | decode_swa = torch.full((2, width), -1, dtype=torch.int32, device=device) |
| 529 | decode_swa[:, :3] = torch.arange(3, dtype=torch.int32, device=device) |
| 530 | compressed = torch.tensor( |
| 531 | [[100, 101, -1, -1]] * 2, dtype=torch.int32, device=device |
| 532 | ) |
| 533 | sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( |
| 534 | decode_swa_indices=decode_swa, |
| 535 | decode_compressed_indices=compressed, |
| 536 | decode_compressed_topk_lens=torch.tensor( |
| 537 | [2, 2], dtype=torch.int32, device=device |
| 538 | ), |
| 539 | prefill_topk_indices=torch.empty((0, 4), dtype=torch.int32, device=device), |
| 540 | query_start_loc=torch.tensor([0, 2], dtype=torch.int32, device=device), |
| 541 | seq_lens=torch.tensor([3], dtype=torch.int32, device=device), |
| 542 | token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32, device=device), |
| 543 | swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), |
| 544 | swa_block_size=64, |
| 545 | compressed_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), |
| 546 | compressed_block_size=64, |
| 547 | window_size=window, |
| 548 | compress_ratio=128, |
| 549 | topk=0, |
| 550 | ) |
| 551 | |
| 552 | assert sparse_indices.cpu().tolist() == [ |
| 553 | # pos 1: window [0, 1]; slot 2 spills to column 4; compressed after it. |
| 554 | [0, 1, -1, -1, 2, 100, 101, -1, -1, -1, -1, -1], |
| 555 | # pos 2: window [0, 1, 2]; nothing spills; compressed at column 4. |
| 556 | [0, 1, 2, -1, 100, 101, -1, -1, -1, -1, -1, -1], |
| 557 | ] |
| 558 | assert sparse_lens.cpu().tolist() == [7, 6] |
| 559 | |
| 560 | |
| 561 | def test_flashinfer_mixed_sparse_indices_noncausal_pad_rows_keep_min_length(): |
| 562 | """A CUDA-graph pad token of the DSpark graph (a zero-length request) gets |
| 563 | a causal pad row's shape instead of negative spill arithmetic.""" |
| 564 | from vllm.models.deepseek_v4.common.ops.cache_utils import ( |
| 565 | build_flashinfer_mixed_sparse_indices, |
| 566 | ) |
| 567 | |
| 568 | device = torch.device("cuda") |
| 569 | window, width = 128, 192 |
| 570 | decode_swa = torch.full((3, width), -1, dtype=torch.int32, device=device) |
| 571 | decode_swa[:2, :3] = torch.arange(3, dtype=torch.int32, device=device) |
| 572 | sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( |
| 573 | decode_swa_indices=decode_swa, |
| 574 | decode_compressed_indices=None, |
| 575 | decode_compressed_topk_lens=None, |
| 576 | prefill_topk_indices=torch.empty((0, 0), dtype=torch.int32, device=device), |
| 577 | # Request 1 is the padding: no query tokens, seq_len 0. |
| 578 | query_start_loc=torch.tensor([0, 2, 2], dtype=torch.int32, device=device), |
| 579 | seq_lens=torch.tensor([3, 0], dtype=torch.int32, device=device), |
| 580 | token_to_req_indices=torch.tensor([0, 0, 1], dtype=torch.int32, device=device), |
| 581 | swa_block_table=torch.zeros((2, 1), dtype=torch.int32, device=device), |
| 582 | swa_block_size=64, |
| 583 | compressed_block_table=None, |
| 584 | compressed_block_size=64, |
| 585 | window_size=window, |
| 586 | compress_ratio=1, |
| 587 | topk=0, |
| 588 | ) |
| 589 | |
| 590 | assert sparse_lens.cpu().tolist() == [129, 128, 128] |
| 591 | assert (sparse_indices[2] == -1).all() |
| 592 | |
| 593 | |
| 594 | @pytest.mark.parametrize("model", ["deepseek_v4", "deepseek_v41"]) |
| 595 | @pytest.mark.parametrize("page_padding", [0, 64]) |
| 596 | def test_flashinfer_sparse_forward_reads_packed_kv_rows(model, page_padding): |
| 597 | """Decode and prefill must read storage rows after vLLM remaps packed pages.""" |
| 598 | import importlib |
| 599 | |
| 600 | from vllm.platforms import current_platform |
| 601 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 602 | |
| 603 | if not current_platform.is_device_capability_family(100): |
| 604 | pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103") |
| 605 | mod = importlib.import_module(f"vllm.models.{model}.nvidia.flashinfer_sparse") |
| 606 | device = "cuda" |
| 607 | block_size = 64 |
| 608 | storage = torch.full( |
| 609 | (3, block_size + page_padding, 512), 99, dtype=torch.bfloat16, device=device |
| 610 | ) |
| 611 | cache = storage[:, :block_size] |
| 612 | cache[1].fill_(2) |
| 613 | cache[2].fill_(3) |
| 614 | attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention) |
| 615 | attn.compress_ratio = 1 |
| 616 | attn.window_size = 128 |
| 617 | attn.scale = 512**-0.5 |
| 618 | attn.kv_cache_torch_dtype = torch.bfloat16 |
| 619 | attn.attn_sink = torch.full((64,), -float("inf"), device=device) |
| 620 | attn.topk_indices_buffer = torch.empty((3, 0), dtype=torch.int32, device=device) |
| 621 | decode_indices = torch.full((1, 128), -1, dtype=torch.int32, device=device) |
| 622 | decode_indices[0, :4] = torch.arange(64, 68, dtype=torch.int32, device=device) |
| 623 | query_start = torch.tensor([0, 1, 3], dtype=torch.int32) |
| 624 | metadata = DeepseekSparseSWAMetadata( |
| 625 | block_table=torch.tensor([[1], [2]], dtype=torch.int32, device=device), |
| 626 | slot_mapping=torch.tensor([67, 130, 131], device=device), |
| 627 | block_size=block_size, |
| 628 | seq_lens=torch.tensor([4, 4], dtype=torch.int32, device=device), |
| 629 | query_start_loc=query_start.to(device), |
| 630 | query_start_loc_cpu=query_start, |
| 631 | token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32, device=device), |
| 632 | decode_swa_indices=decode_indices, |
| 633 | decode_swa_lens=torch.tensor([4], dtype=torch.int32, device=device), |
| 634 | decode_swa_width=128, |
| 635 | is_valid_token=torch.ones(3, dtype=torch.bool, device=device), |
| 636 | replay_start=torch.zeros(2, dtype=torch.int32, device=device), |
| 637 | num_decodes=1, |
| 638 | num_prefills=1, |
| 639 | num_decode_tokens=1, |
| 640 | num_prefill_tokens=2, |
| 641 | max_decode_query_len=1, |
| 642 | ) |
| 643 | query = torch.zeros((3, 64, 512), dtype=torch.bfloat16, device=device) |
| 644 | output = torch.empty_like(query) |
| 645 | attn._forward(query, None, cache, metadata, None, True, output) |
| 646 | expected = torch.tensor([2, 3, 3], dtype=output.dtype, device=device) |
| 647 | torch.testing.assert_close(output, expected[:, None, None].expand_as(output)) |
| 648 | |
| 649 | |
| 650 | def _make_rope_quant_attn(mod, num_rows: int, device: str): |
| 651 | """A bare DSv4 FlashInfer layer set up for RopeQuant: 128 heads, fp8 KV.""" |
| 652 | from types import SimpleNamespace |
| 653 | |
| 654 | heads, groups, rank = 128, 16, 256 |
| 655 | angles = torch.arange(2048, device=device, dtype=torch.float32)[:, None] * 0.05 |
| 656 | angles = angles + torch.arange(32, device=device)[None] * 0.1 |
| 657 | attn = object.__new__(mod.DeepseekV4FlashInferMLAAttention) |
| 658 | attn.compress_ratio = 1 |
| 659 | attn.window_size = 128 |
| 660 | attn.scale = 512**-0.5 |
| 661 | attn.kv_cache_torch_dtype = torch.float8_e4m3fn |
| 662 | attn._flashinfer_fp8_bmm1_scale = attn.scale |
| 663 | attn._flashinfer_fp8_bmm2_scale = 1.0 |
| 664 | attn.attn_sink = torch.linspace(-1, 1, heads, device=device) |
| 665 | attn.padded_heads = heads |
| 666 | attn.topk_indices_buffer = torch.empty( |
| 667 | (num_rows, 0), dtype=torch.int32, device=device |
| 668 | ) |
| 669 | attn.rotary_emb = SimpleNamespace( |
| 670 | cos_sin_cache=torch.cat((angles.cos(), angles.sin()), -1).contiguous() |
| 671 | ) |
| 672 | attn.n_local_heads, attn.n_local_groups, attn.o_lora_rank = heads, groups, rank |
| 673 | attn.nope_head_dim, attn.rope_head_dim = 448, 64 |
| 674 | attn._einsum_recipe, attn._tma_aligned_scales = (1, 1, 128), True |
| 675 | attn.wo_a = SimpleNamespace( |
| 676 | weight=(torch.randn((groups, rank, 4096), device=device) * 0.05).to( |
| 677 | torch.float8_e4m3fn |
| 678 | ), |
| 679 | weight_scale=torch.ones((groups, rank, 32), device=device), |
| 680 | ) |
| 681 | attn.wo_b = lambda z: z |
| 682 | return attn |
| 683 | |
| 684 | |
| 685 | def _rope_quant_vs_unfused(attn, q, cache, metadata, positions, num_tokens): |
| 686 | """Run `_forward` unfused and fused; return (bf16 attention, z_unfused, z_fused).""" |
| 687 | from vllm.models.deepseek_v4.nvidia.ops.o_proj import rope_quant_attn_out |
| 688 | |
| 689 | heads, groups = attn.n_local_heads, attn.n_local_groups |
| 690 | unfused = torch.empty(q.shape, dtype=torch.bfloat16, device=q.device) |
| 691 | attn._forward(q, None, cache, metadata, None, True, unfused) |
| 692 | fused = rope_quant_attn_out(q.shape[0], q.device) |
| 693 | metadata.flashinfer_sparse_index_cache.clear() |
| 694 | attn._forward(q, None, cache, metadata, None, True, fused) |
| 695 | z_unfused = attn._o_proj(unfused[:num_tokens, :heads], positions) |
| 696 | z_fused = attn._o_proj(fused, positions)[:num_tokens] |
| 697 | return unfused[:num_tokens], z_unfused, z_fused.view(num_tokens, groups, -1) |
| 698 | |
| 699 | |
| 700 | def _assert_projects_alike(z_unfused, z_fused): |
| 701 | z_unfused = z_unfused.float().view_as(z_fused) |
| 702 | rel = (z_fused.float() - z_unfused).norm() / z_unfused.norm() |
| 703 | assert rel < 2e-2, rel |
| 704 | |
| 705 | |
| 706 | def _skip_unless_rope_quant(): |
| 707 | from vllm.platforms import current_platform |
| 708 | |
| 709 | if not current_platform.is_device_capability_family(100): |
| 710 | pytest.skip("Requires FlashInfer TRTLLM sparse MLA on SM100/SM103") |
| 711 | |
| 712 | |
| 713 | def test_flashinfer_rope_quant_matches_unfused_o_proj(): |
| 714 | """RopeQuant's fused output must project like the bf16 + inv-RoPE/quant path. |
| 715 | |
| 716 | One mixed step (two decodes, a three-token prefill) plus CUDA-graph pad rows, |
| 717 | so both calls run over the full buffer with un-rebased cum_seq_lens_q. |
| 718 | """ |
| 719 | from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod |
| 720 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 721 | |
| 722 | _skip_unless_rope_quant() |
| 723 | torch.manual_seed(0) |
| 724 | device = "cuda" |
| 725 | block_size = 64 |
| 726 | q_lens, seq_lens = [1, 1, 3], [5, 9, 70] |
| 727 | num_tokens, num_rows = sum(q_lens), sum(q_lens) + 3 |
| 728 | cache = torch.randn((3, block_size, 512), device=device).to(torch.float8_e4m3fn) |
| 729 | attn = _make_rope_quant_attn(mod, num_rows, device) |
| 730 | |
| 731 | decode_indices = torch.full((2, 128), -1, dtype=torch.int32, device=device) |
| 732 | for i, seq_len in enumerate(seq_lens[:2]): |
| 733 | decode_indices[i, :seq_len] = torch.arange( |
| 734 | i * block_size, i * block_size + seq_len, device=device |
| 735 | ) |
| 736 | query_start = torch.tensor([0, 1, 2, 5], dtype=torch.int32) |
| 737 | metadata = DeepseekSparseSWAMetadata( |
| 738 | block_table=torch.tensor([[0], [1], [2]], dtype=torch.int32, device=device), |
| 739 | slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device), |
| 740 | block_size=block_size, |
| 741 | seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=device), |
| 742 | query_start_loc=query_start.to(device), |
| 743 | query_start_loc_cpu=query_start, |
| 744 | token_to_req_indices=torch.tensor( |
| 745 | [0, 1, 2, 2, 2], dtype=torch.int32, device=device |
| 746 | ), |
| 747 | decode_swa_indices=decode_indices, |
| 748 | decode_swa_lens=torch.tensor(seq_lens[:2], dtype=torch.int32, device=device), |
| 749 | decode_swa_width=128, |
| 750 | is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device), |
| 751 | num_decodes=2, |
| 752 | num_prefills=1, |
| 753 | num_decode_tokens=2, |
| 754 | num_prefill_tokens=3, |
| 755 | max_decode_query_len=1, |
| 756 | ) |
| 757 | positions = torch.tensor([4, 8, 67, 68, 69], device=device) |
| 758 | q = torch.randn((num_rows, 128, 512), device=device).to(torch.float8_e4m3fn) |
| 759 | _, z_unfused, z_fused = _rope_quant_vs_unfused( |
| 760 | attn, q, cache, metadata, positions, num_tokens |
| 761 | ) |
| 762 | _assert_projects_alike(z_unfused, z_fused) |
| 763 | |
| 764 | |
| 765 | @pytest.mark.parametrize("context_len", [0, 20, 127, 900]) |
| 766 | def test_flashinfer_dspark_noncausal_block_sees_future_tokens(context_len): |
| 767 | """Every DSpark draft token attends to the whole block, fused or not. |
| 768 | |
| 769 | The kernel counts every entry of its active index ranges as a key, so the |
| 770 | builder must place the block around each query's causal window without -1 |
| 771 | gaps, down to an empty context. |
| 772 | """ |
| 773 | from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as mod |
| 774 | from vllm.v1.attention.backends.mla.compressor_utils import ( |
| 775 | get_dspark_swa_index_width, |
| 776 | ) |
| 777 | from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata |
| 778 | |
| 779 | _skip_unless_rope_quant() |
| 780 | torch.manual_seed(0) |
| 781 | device = "cuda" |
| 782 | block_size, window, draft = 64, 128, 4 |
| 783 | width = get_dspark_swa_index_width(window, draft) |
| 784 | contexts = [context_len, context_len + 37] |
| 785 | num_reqs, num_tokens = len(contexts), len(contexts) * draft |
| 786 | num_rows = num_tokens + 2 |
| 787 | pages = -(-(max(contexts) + draft) // block_size) |
| 788 | cache = torch.randn((num_reqs * pages, block_size, 512), device=device) |
| 789 | cache = cache.clamp_(-1, 1).to(torch.float8_e4m3fn) |
| 790 | attn = _make_rope_quant_attn(mod, num_rows, device) |
| 791 | attn.attn_sink = torch.full((128,), -float("inf"), device=device) |
| 792 | |
| 793 | # As the DSpark SWA builder lays them out: the trailing window of context |
| 794 | # and the whole block, contiguous from column 0. |
| 795 | decode_indices = torch.full((num_tokens, width), -1, dtype=torch.int32) |
| 796 | visible, positions = [], [] |
| 797 | for r, ctx in enumerate(contexts): |
| 798 | slots = torch.arange(max(ctx - window, 0), ctx + draft) + r * pages * block_size |
| 799 | for k in range(draft): |
| 800 | decode_indices[r * draft + k, : slots.numel()] = slots.to(torch.int32) |
| 801 | visible.append(slots) |
| 802 | positions.append(ctx + k) |
| 803 | query_start = torch.arange(0, num_tokens + 1, draft, dtype=torch.int32) |
| 804 | seq_lens = torch.tensor([c + draft for c in contexts], dtype=torch.int32) |
| 805 | metadata = DeepseekSparseSWAMetadata( |
| 806 | block_table=torch.arange( |
| 807 | num_reqs * pages, dtype=torch.int32, device=device |
| 808 | ).view(num_reqs, pages), |
| 809 | slot_mapping=torch.zeros(num_tokens, dtype=torch.int64, device=device), |
| 810 | block_size=block_size, |
| 811 | seq_lens=seq_lens.to(device), |
| 812 | query_start_loc=query_start.to(device), |
| 813 | query_start_loc_cpu=query_start, |
| 814 | token_to_req_indices=torch.arange( |
| 815 | num_reqs, dtype=torch.int32, device=device |
| 816 | ).repeat_interleave(draft), |
| 817 | decode_swa_indices=decode_indices.to(device), |
| 818 | decode_swa_lens=torch.tensor( |
| 819 | [v.numel() for v in visible], dtype=torch.int32, device=device |
| 820 | ), |
| 821 | decode_swa_width=width, |
| 822 | is_valid_token=torch.ones(num_tokens, dtype=torch.bool, device=device), |
| 823 | num_decodes=num_reqs, |
| 824 | num_prefills=0, |
| 825 | num_decode_tokens=num_tokens, |
| 826 | num_prefill_tokens=0, |
| 827 | max_decode_query_len=draft, |
| 828 | ) |
| 829 | q = torch.randn((num_rows, 128, 512), device=device).clamp_(-1, 1) |
| 830 | q = q.to(torch.float8_e4m3fn) |
| 831 | attention, z_unfused, z_fused = _rope_quant_vs_unfused( |
| 832 | attn, q, cache, metadata, torch.tensor(positions, device=device), num_tokens |
| 833 | ) |
| 834 | |
| 835 | keys_all = cache.flatten(0, 1).float() |
| 836 | for token, slots in enumerate(visible): |
| 837 | keys = keys_all[slots.to(device)] |
| 838 | weights = torch.softmax(q[token].float() @ keys.T * attn.scale, -1) |
| 839 | torch.testing.assert_close( |
| 840 | attention[token].float(), weights @ keys, atol=0.05, rtol=0.05 |
| 841 | ) |
| 842 | _assert_projects_alike(z_unfused, z_fused) |